diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json new file mode 100644 index 000000000..14bf1024e --- /dev/null +++ b/.basedpyright/baseline.json @@ -0,0 +1,966 @@ +{ + "files": { + "./git/cmd.py": [ + { + "code": "reportGeneralTypeIssues", + "range": { + "startColumn": 13, + "endColumn": 32, + "lineCount": 1 + } + }, + { + "code": "reportOptionalOperand", + "range": { + "startColumn": 27, + "endColumn": 35, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 28, + "endColumn": 31, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 28, + "endColumn": 40, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 19, + "endColumn": 68, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 19, + "endColumn": 31, + "lineCount": 1 + } + } + ], + "./git/config.py": [ + { + "code": "reportGeneralTypeIssues", + "range": { + "startColumn": 11, + "endColumn": 26, + "lineCount": 1 + } + }, + { + "code": "reportInvalidTypeVarUse", + "range": { + "startColumn": 43, + "endColumn": 45, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 41, + "endColumn": 48, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 32, + "endColumn": 39, + "lineCount": 1 + } + }, + { + "code": "reportCallIssue", + "range": { + "startColumn": 25, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 30, + "endColumn": 39, + "lineCount": 1 + } + } + ], + "./git/db.py": [ + { + "code": "reportIncompatibleMethodOverride", + "range": { + "startColumn": 8, + "endColumn": 12, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 61, + "endColumn": 79, + "lineCount": 1 + } + }, + { + "code": "reportIncompatibleMethodOverride", + "range": { + "startColumn": 8, + "endColumn": 14, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 70, + "endColumn": 88, + "lineCount": 1 + } + } + ], + "./git/index/base.py": [ + { + "code": "reportArgumentType", + "range": { + "startColumn": 30, + "endColumn": 36, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 28, + "endColumn": 34, + "lineCount": 1 + } + }, + { + "code": "reportAssignmentType", + "range": { + "startColumn": 38, + "endColumn": 76, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 45, + "endColumn": 53, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 60, + "endColumn": 63, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 56, + "endColumn": 60, + "lineCount": 1 + } + }, + { + "code": "reportSelfClsParameterName", + "range": { + "startColumn": 30, + "endColumn": 33, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 51, + "endColumn": 57, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 55, + "endColumn": 61, + "lineCount": 1 + } + }, + { + "code": "reportIncompatibleMethodOverride", + "range": { + "startColumn": 8, + "endColumn": 12, + "lineCount": 1 + } + }, + { + "code": "reportAssignmentType", + "range": { + "startColumn": 20, + "endColumn": 46, + "lineCount": 1 + } + } + ], + "./git/index/fun.py": [ + { + "code": "reportArgumentType", + "range": { + "startColumn": 8, + "endColumn": 36, + "lineCount": 1 + } + } + ], + "./git/index/typ.py": [ + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 37, + "endColumn": 46, + "lineCount": 1 + } + } + ], + "./git/objects/base.py": [ + { + "code": "reportArgumentType", + "range": { + "startColumn": 20, + "endColumn": 27, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 29, + "endColumn": 36, + "lineCount": 1 + } + } + ], + "./git/objects/blob.py": [ + { + "code": "reportIncompatibleVariableOverride", + "range": { + "startColumn": 4, + "endColumn": 8, + "lineCount": 1 + } + } + ], + "./git/objects/commit.py": [ + { + "code": "reportIncompatibleVariableOverride", + "range": { + "startColumn": 4, + "endColumn": 8, + "lineCount": 1 + } + }, + { + "code": "reportIncompatibleMethodOverride", + "range": { + "startColumn": 8, + "endColumn": 31, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 20, + "endColumn": 33, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 20, + "endColumn": 30, + "lineCount": 1 + } + } + ], + "./git/objects/submodule/base.py": [ + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 19, + "endColumn": 24, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 19, + "endColumn": 24, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 23, + "endColumn": 25, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 23, + "endColumn": 25, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 41, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 52, + "endColumn": 84, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 20, + "endColumn": 40, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 35, + "endColumn": 42, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 15, + "endColumn": 20, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 15, + "endColumn": 20, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 19, + "endColumn": 33, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 19, + "endColumn": 33, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 15, + "endColumn": 27, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 15, + "endColumn": 27, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 15, + "endColumn": 19, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 15, + "endColumn": 19, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 18, + "endColumn": 20, + "lineCount": 1 + } + } + ], + "./git/objects/tag.py": [ + { + "code": "reportIncompatibleVariableOverride", + "range": { + "startColumn": 4, + "endColumn": 8, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 72, + "endColumn": 78, + "lineCount": 1 + } + } + ], + "./git/objects/tree.py": [ + { + "code": "reportIncompatibleVariableOverride", + "range": { + "startColumn": 4, + "endColumn": 8, + "lineCount": 1 + } + }, + { + "code": "reportIncompatibleMethodOverride", + "range": { + "startColumn": 8, + "endColumn": 31, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 19, + "endColumn": 83, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 15, + "endColumn": 54, + "lineCount": 1 + } + } + ], + "./git/objects/util.py": [ + { + "code": "reportAssignmentType", + "range": { + "startColumn": 23, + "endColumn": 34, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 22, + "endColumn": 26, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 30, + "endColumn": 34, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 15, + "endColumn": 54, + "lineCount": 1 + } + } + ], + "./git/refs/log.py": [ + { + "code": "reportArgumentType", + "range": { + "startColumn": 30, + "endColumn": 34, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 17, + "endColumn": 22, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 28, + "endColumn": 30, + "lineCount": 1 + } + } + ], + "./git/refs/reference.py": [ + { + "code": "reportInvalidTypeVarUse", + "range": { + "startColumn": 22, + "endColumn": 34, + "lineCount": 1 + } + }, + { + "code": "reportIncompatibleVariableOverride", + "range": { + "startColumn": 13, + "endColumn": 17, + "lineCount": 1 + } + } + ], + "./git/refs/symbolic.py": [ + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 15, + "endColumn": 20, + "lineCount": 1 + } + } + ], + "./git/refs/tag.py": [ + { + "code": "reportIncompatibleMethodOverride", + "range": { + "startColumn": 8, + "endColumn": 14, + "lineCount": 1 + } + }, + { + "code": "reportIncompatibleMethodOverride", + "range": { + "startColumn": 8, + "endColumn": 14, + "lineCount": 1 + } + } + ], + "./git/remote.py": [ + { + "code": "reportArgumentType", + "range": { + "startColumn": 25, + "endColumn": 36, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 29, + "endColumn": 40, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 26, + "endColumn": 38, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 26, + "endColumn": 38, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 26, + "endColumn": 38, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 26, + "endColumn": 38, + "lineCount": 1 + } + } + ], + "./git/repo/base.py": [ + { + "code": "reportRedeclaration", + "range": { + "startColumn": 4, + "endColumn": 15, + "lineCount": 1 + } + }, + { + "code": "reportAttributeAccessIssue", + "range": { + "startColumn": 18, + "endColumn": 22, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 35, + "endColumn": 41, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 15, + "endColumn": 46, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 15, + "endColumn": 51, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 15, + "endColumn": 28, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 16, + "endColumn": 31, + "lineCount": 1 + } + }, + { + "code": "reportTypedDictNotRequiredAccess", + "range": { + "startColumn": 21, + "endColumn": 31, + "lineCount": 1 + } + }, + { + "code": "reportTypedDictNotRequiredAccess", + "range": { + "startColumn": 34, + "endColumn": 44, + "lineCount": 1 + } + }, + { + "code": "reportTypedDictNotRequiredAccess", + "range": { + "startColumn": 65, + "endColumn": 79, + "lineCount": 1 + } + }, + { + "code": "reportTypedDictNotRequiredAccess", + "range": { + "startColumn": 82, + "endColumn": 102, + "lineCount": 1 + } + }, + { + "code": "reportTypedDictNotRequiredAccess", + "range": { + "startColumn": 50, + "endColumn": 69, + "lineCount": 1 + } + }, + { + "code": "reportTypedDictNotRequiredAccess", + "range": { + "startColumn": 68, + "endColumn": 85, + "lineCount": 1 + } + }, + { + "code": "reportTypedDictNotRequiredAccess", + "range": { + "startColumn": 88, + "endColumn": 111, + "lineCount": 1 + } + }, + { + "code": "reportTypedDictNotRequiredAccess", + "range": { + "startColumn": 51, + "endColumn": 73, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 12, + "endColumn": 26, + "lineCount": 1 + } + }, + { + "code": "reportAssignmentType", + "range": { + "startColumn": 22, + "endColumn": 38, + "lineCount": 1 + } + } + ], + "./git/repo/fun.py": [ + { + "code": "reportArgumentType", + "range": { + "startColumn": 25, + "endColumn": 33, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 35, + "endColumn": 43, + "lineCount": 1 + } + }, + { + "code": "reportAssignmentType", + "range": { + "startColumn": 18, + "endColumn": 28, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 11, + "endColumn": 14, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 24, + "endColumn": 27, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 11, + "endColumn": 14, + "lineCount": 1 + } + }, + { + "code": "reportReturnType", + "range": { + "startColumn": 11, + "endColumn": 20, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 25, + "endColumn": 28, + "lineCount": 1 + } + }, + { + "code": "reportArgumentType", + "range": { + "startColumn": 24, + "endColumn": 27, + "lineCount": 1 + } + } + ], + "./test/deprecation/test_basic.py": [ + { + "code": "reportUnusedExpression", + "range": { + "startColumn": 12, + "endColumn": 62, + "lineCount": 1 + } + } + ] + } +} diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 1cc049a69..000000000 --- a/.flake8 +++ /dev/null @@ -1,26 +0,0 @@ -[flake8] - -show-source = True -count = True -statistics = True - -# E266 = too many leading '#' for block comment -# E731 = do not assign a lambda expression, use a def -# TC002 = move third party import to TYPE_CHECKING -# TC, TC2 = flake8-type-checking - -# select = C,E,F,W ANN, TC, TC2 # to enable code. Disabled if not listed, including builtin codes -enable-extensions = TC, TC2 # only needed for extensions not enabled by default - -ignore = E266, E731 - -exclude = .tox, .venv, build, dist, doc, git/ext/ - -rst-roles = # for flake8-RST-docstrings - attr, class, func, meth, mod, obj, ref, term, var # used by sphinx - -min-python-version = 3.7.0 - -# for `black` compatibility -max-line-length = 120 -extend-ignore = E203, W503 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2fe73ca77..7f803ee43 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,11 +1,15 @@ version: 2 +enable-beta-ecosystems: true updates: -- package-ecosystem: "github-actions" +- package-ecosystem: github-actions directory: "/" schedule: - interval: "weekly" + interval: weekly -- package-ecosystem: "gitsubmodule" +- package-ecosystem: pre-commit directory: "/" schedule: - interval: "weekly" + interval: monthly + groups: + pre-commit: + patterns: ["*"] diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml new file mode 100644 index 000000000..b10336a15 --- /dev/null +++ b/.github/workflows/alpine-test.yml @@ -0,0 +1,97 @@ +name: test-alpine + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + + container: + image: alpine:latest + + defaults: + run: + shell: sudo -u runner sh -exo pipefail {0} + + steps: + - name: Prepare Alpine Linux + run: | + apk add sudo git git-daemon python3 py3-pip py3-virtualenv + echo 'Defaults env_keep += "CI GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env + addgroup -g 127 docker + adduser -D -u 1001 runner # TODO: Check if this still works on GHA as intended. + adduser runner docker + shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. + + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set workspace ownership + run: | + chown -R runner:docker -- "$GITHUB_WORKSPACE" + shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. + + - name: Prepare this repo for tests + run: | + ./init-tests-after-clone.sh + + - name: Set git user identity and command aliases for the tests + run: | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + # If we rewrite the user's config by accident, we will mess it up + # and cause subsequent tests to fail + cat test/fixtures/.gitconfig >> ~/.gitconfig + + - name: Set up virtual environment + run: | + python -m venv .venv + + - name: Update PyPA packages + run: | + . .venv/bin/activate + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel + + - name: Install project and test dependencies + run: | + . .venv/bin/activate + pip install ./smmap ./gitdb '.[test]' + + - name: Show POSIX file ownership + run: | + ls -ld -- \ + "$(pwd)" \ + "$(pwd)/.git" \ + "${HOME:?HOME is not set}/.gitconfig" \ + 2>&1 || true + + - name: Show safe.directory entries + # `actions/checkout`'s safe.directory add is only durable for the + # checkout itself (it writes under a throwaway HOME override and + # then discards it), so by the time this step runs the runner + # user's `~/.gitconfig` has no entries -- and the Alpine container + # chowns the workspace to runner:docker to match the test user, so + # git accepts the ownership without one. Expected: `(none)`. + run: git config --global --get-all safe.directory || echo "(none)" + + - name: Show version and platform information + run: | + . .venv/bin/activate + uname -a + command -v git python + git version + python --version + python -c 'import os, sys; print(f"sys.platform={sys.platform!r}, os.name={os.name!r}")' + + - name: Test with pytest + run: | + . .venv/bin/activate + pytest --color=yes -p no:sugar --instafail -vv diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ae5241898..eeb68cd0c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,44 +13,59 @@ name: "CodeQL" on: push: + branches: [main] pull_request: schedule: - cron: '27 10 * * 3' jobs: analyze: - name: Analyze + name: Analyze (${{ matrix.language }}) # Runner size impacts CodeQL analysis time. To learn more, please see: # - https://gh.io/recommended-hardware-resources-for-running-codeql # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} permissions: - actions: read - contents: read security-events: write strategy: fail-fast: false matrix: - language: [ 'python' ] - # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] - # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - + include: + - language: actions + build-mode: none + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 + + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - setup-python-dependencies: false + build-mode: ${{ matrix.build-mode }} + config: | + paths-ignore: + - gitdb/gitdb/test/** + - smmap/smmap/test/** # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. @@ -58,23 +73,23 @@ jobs: # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index f3937d21e..b8150e93d 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -1,14 +1,29 @@ name: test-cygwin -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: -jobs: - build: - runs-on: windows-latest +permissions: + contents: read +jobs: + test: strategy: + matrix: + selection: [fast, perf] + include: + - selection: fast + additional-pytest-args: --ignore=test/performance + - selection: perf + additional-pytest-args: test/performance + fail-fast: false + runs-on: windows-latest + env: CHERE_INVOKING: "1" CYGWIN_NOWINPATH: "1" @@ -21,16 +36,16 @@ jobs: - name: Force LF line endings run: | git config --global core.autocrlf false # Affects the non-Cygwin git. - shell: bash # Use Git Bash instead of Cygwin Bash for this step. + shell: pwsh # Do this outside Cygwin, to affect actions/checkout. - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Install Cygwin - uses: cygwin/cygwin-install-action@v4 + uses: cygwin/cygwin-install-action@v6 with: - packages: python39 python39-pip python39-virtualenv git + packages: git python39 python-pip-wheel python-setuptools-wheel python-wheel-wheel add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output @@ -41,6 +56,7 @@ jobs: - name: Special configuration for Cygwin git run: | git config --global --add safe.directory "$(pwd)" + git config --global --add safe.directory "$(pwd)/.git" git config --global core.autocrlf false - name: Prepare this repo for tests @@ -55,19 +71,58 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Ensure the "pip" command is available + - name: Set up virtual environment run: | - # This is used unless, and before, an updated pip is installed. - ln -s pip3 /usr/bin/pip + python3.9 -m venv .venv + echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" - name: Update PyPA packages run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies run: | - pip install ".[test]" + pip install ./smmap ./gitdb '.[test]' + + - name: Show POSIX file ownership + # Cygwin's `ls -ld` reports the NTFS Owner SID via Cygwin's SID-to-uid + # mapping (well-known SIDs by their RID, machine-local accounts by + # 0x30000+RID). That mapping is what Cygwin git's + # `is_path_owned_by_current_user` reduces to, so this is the view that + # determines whether `safe.directory` is consulted. + run: | + ls -ld -- \ + "$(pwd)" \ + "$(pwd)/.git" \ + "${HOME:?HOME is not set}/.gitconfig" \ + 2>&1 || true + + - name: Show NTFS file ownership + # Authoritative NTFS Owner via Get-Acl, with no Cygwin SID-to-uid layer + # in between -- useful for confirming what the Cygwin view reports as + # "Administrators" is the BUILTIN\Administrators SID (S-1-5-32-544). + shell: pwsh + run: | + $paths = @( + "$pwd", + "$pwd\.git", + "$env:USERPROFILE\.gitconfig" + ) + foreach ($p in $paths) { + if (Test-Path -LiteralPath $p) { + try { + $owner = (Get-Acl -LiteralPath $p).Owner + } catch { + $owner = "ERROR: $($_.Exception.Message)" + } + "{0,-44} {1}" -f $owner, $p + } else { + "(missing: $p)" + } + } + + - name: Show safe.directory entries + run: git config --global --get-all safe.directory - name: Show version and platform information run: | @@ -77,6 +132,6 @@ jobs: python --version python -c 'import os, sys; print(f"sys.platform={sys.platform!r}, os.name={os.name!r}")' - - name: Test with pytest + - name: Test with pytest (${{ matrix.additional-pytest-args }}) run: | - pytest --color=yes -p no:sugar --instafail -vv + pytest --color=yes -p no:sugar --instafail -vv ${{ matrix.additional-pytest-args }} diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 000000000..4803e733b --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,42 @@ +name: Dependency packages + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ matrix.python-version == '3.7' && 'ubuntu-22.04' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: + project: [smmap, gitdb] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest flake8 ./smmap + if test "${{ matrix.project }}" = gitdb; then python -m pip install ./gitdb; fi + - name: Lint + run: flake8 "${{ matrix.project }}/${{ matrix.project }}" --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Test + run: | + if test "${{ matrix.project }}" = gitdb; then + GITDB_TEST_GIT_REPO_BASE="$(git rev-parse --git-common-dir)" pytest -o addopts= -v gitdb/gitdb/test + else + pytest -o addopts= -v smmap/smmap/test + fi diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f9c5b70b3..d22640951 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,20 +1,23 @@ name: Lint -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.x" - - uses: pre-commit/action@v3.0.0 - with: - extra_args: --all-files --hook-stage manual - env: - SKIP: black-format + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 08ff4efdf..7dfa2e8f2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -3,43 +3,68 @@ name: Python package -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: permissions: contents: read jobs: - build: + test: strategy: - fail-fast: false matrix: - os: ["ubuntu-latest", "macos-13", "windows-latest"] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + os-type: [ubuntu, macos, windows] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"] + exclude: + - os-type: macos + python-version: "3.7" # Not available for the ARM-based macOS runners. + - os-type: macos + python-version: "3.14t" + - os-type: macos + python-version: "3.15t" include: + - os-ver: latest + - os-type: ubuntu + python-version: "3.7" + os-ver: "22.04" + - build-docs: true # We ensure documentation builds, except on very old interpreters. + - python-version: "3.7" + build-docs: false + - python-version: "3.8" + build-docs: false - experimental: false + - python-version: "3.15" + experimental: true + - python-version: "3.15t" + experimental: true + + fail-fast: false - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os-type }}-${{ matrix.os-ver }} defaults: run: shell: bash --noprofile --norc -exo pipefail {0} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} - name: Set up WSL (Windows) - if: startsWith(matrix.os, 'windows') - uses: Vampire/setup-wsl@v2.0.2 + if: matrix.os-type == 'windows' + uses: Vampire/setup-wsl@v7.0.0 with: - distribution: Debian + wsl-version: 1 - name: Prepare this repo for tests run: | @@ -55,12 +80,59 @@ jobs: - name: Update PyPA packages run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies run: | - pip install ".[test]" + pip install ./smmap ./gitdb '.[test]' + + - name: Show POSIX file ownership + # Linux and macOS only. On Windows, Git Bash's `ls -ld` reports a + # uniform uid+gid for every path regardless of NTFS Owner (MSYS2's + # SID-to-uid mapping doesn't have Cygwin's fidelity), so it would + # not be informative here. The NTFS Owner check below covers Windows. + if: matrix.os-type != 'windows' + run: | + ls -ld -- \ + "$(pwd)" \ + "$(pwd)/.git" \ + "${HOME:?HOME is not set}/.gitconfig" \ + 2>&1 || true + + - name: Show NTFS file ownership + # Windows only. Reads NTFS Owner directly via Get-Acl, which is the + # authoritative view for Windows-side ownership questions; the POSIX + # view via Git Bash's MSYS2 layer is not a reliable proxy here. + if: matrix.os-type == 'windows' + shell: pwsh + run: | + $paths = @( + "$pwd", + "$pwd\.git", + "$env:USERPROFILE\.gitconfig" + ) + foreach ($p in $paths) { + if (Test-Path -LiteralPath $p) { + try { + $owner = (Get-Acl -LiteralPath $p).Owner + } catch { + $owner = "ERROR: $($_.Exception.Message)" + } + "{0,-44} {1}" -f $owner, $p + } else { + "(missing: $p)" + } + } + + - name: Show safe.directory entries + # `actions/checkout`'s safe.directory add is only durable for the + # checkout itself (it writes under a throwaway HOME override and + # then discards it), so by the time this step runs the runner + # user's `~/.gitconfig` has no entries -- and git accepts the + # workspace's ownership anyway: Git for Windows via its + # Admins-group exemption on the windows matrix; on Linux/macOS + # the workspace is owned by the test user. Expected: `(none)`. + run: git config --global --get-all safe.directory || echo "(none)" - name: Show version and platform information run: | @@ -72,7 +144,7 @@ jobs: # For debugging hook tests on native Windows systems that may have WSL. - name: Show bash.exe candidates (Windows) - if: startsWith(matrix.os, 'windows') + if: matrix.os-type == 'windows' run: | set +e bash.exe -c 'printenv WSL_DISTRO_NAME; uname -a' @@ -80,11 +152,18 @@ jobs: continue-on-error: true - name: Check types with mypy + if: matrix.python-version != '3.7' && matrix.python-version != '3.8' run: | - mypy -p git - # With new versions of mypy new issues might arise. This is a problem if there is nobody able to fix them, - # so we have to ignore errors until that changes. - continue-on-error: true + mypy --python-version="${PYTHON_VERSION%t}" # Version only, with no "t" for free-threaded. + env: + MYPY_FORCE_COLOR: "1" + TERM: "xterm-256color" # For color: https://github.com/python/mypy/issues/13817 + PYTHON_VERSION: ${{ matrix.python-version }} + + - name: Check types with basedpyright + if: matrix.os-type == 'ubuntu' && matrix.python-version == '3.12' + run: | + basedpyright --warnings - name: Test with pytest run: | @@ -92,6 +171,7 @@ jobs: continue-on-error: false - name: Documentation + if: matrix.build-docs run: | - pip install -r doc/requirements.txt + pip install '.[doc]' make -C doc html diff --git a/.gitignore b/.gitignore index 7765293d8..eab294a65 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ __pycache__/ # Transient editor files *.swp *~ +\#*# +.#*# # Editor configuration nbproject @@ -47,3 +49,6 @@ output.txt # Finder metadata .DS_Store + +# Files created by OSS-Fuzz when running locally +fuzz_*.pkg.spec diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 251eeeec4..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "gitdb"] - url = https://github.com/gitpython-developers/gitdb.git - path = git/ext/gitdb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ac5baa00..75a29d6db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,39 +1,40 @@ +exclude: ^(?:gitdb|smmap)/ + repos: -- repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 +- repo: https://github.com/codespell-project/codespell + rev: v2.4.3 hooks: - - id: black - alias: black-check - name: black (check) - args: [--check, --diff] - exclude: ^git/ext/ - stages: [manual] - - - id: black - alias: black-format - name: black (format) - exclude: ^git/ext/ + - id: codespell + additional_dependencies: [tomli] + exclude: ^test/fixtures/ -- repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.0 hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear==23.9.16 - - flake8-comprehensions==3.14.0 - - flake8-typing-imports==1.14.0 - exclude: ^doc|^git/ext/ + - id: ruff-check + args: ["--fix"] + exclude: ^git/ext/ + - id: ruff-format + exclude: ^git/ext/ - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.5 + rev: v0.11.0.1 hooks: - id: shellcheck args: [--color] exclude: ^test/fixtures/polyglot$|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: + - id: end-of-file-fixer + exclude: ^test/fixtures/|COPYING|LICENSE + - id: check-symlinks - id: check-toml - id: check-yaml - id: check-merge-conflict + +- repo: https://github.com/abravalheri/validate-pyproject + rev: v0.25 + hooks: + - id: validate-pyproject diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..9bce80fd2 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,36 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need. +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "doc/" directory with Sphinx. +sphinx: + configuration: doc/source/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub. +formats: all + +# Optional but recommended, declare the Python requirements required +# to build your documentation. +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - method: pip + path: . + extra_requirements: + - doc diff --git a/AUTHORS b/AUTHORS index 3b97c9473..95205f439 100644 --- a/AUTHORS +++ b/AUTHORS @@ -53,5 +53,10 @@ Contributors are: -Santos Gallegos -Wenhan Zhu -Eliah Kagan +-Ethan Lin +-Jonas Scharpf +-Gordon Marx +-Enji Cooper +-Harshita Yadav Portions derived from other open source works and are clearly marked. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e108f1b80..76f276323 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,3 +8,30 @@ The following is a short step-by-step rundown of what one typically would do to - Try to avoid massive commits and prefer to take small steps, with one commit for each. - Feel free to add yourself to AUTHORS file. - Create a pull request. + +## Quality expectations + +Contributions must be made with care and meet the quality bar of the surrounding code. +That means a change should not leave GitPython worse than it was before: it should be +readable, maintainable, tested where practical, documented and consistent with the +existing style and behavior. + +A contribution that works only narrowly but lowers the quality of the +codebase may be declined. The maintainers may not always be able to provide +detailed feedback. + +## Prevent agent impersonation + +AI agents communicating through a person's account must identify themselves, for +example in issue or PR descriptions and comments. AI assistance that does not replace +the person as the speaker, such as proofreading or wording polish, does not require +identification. + +Attributing AI assistance in commit metadata, for example with a `Co-authored-by` +trailer, is welcome but not required. Code is reviewed the same way regardless of its +origin. + +## Fuzzing Test Specific Documentation + +For details related to contributing to the fuzzing test suite and OSS-Fuzz integration, please +refer to the dedicated [fuzzing README](./fuzzing/README.md). diff --git a/Makefile b/Makefile index 839dc9f78..d4f9acf87 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,8 @@ -.PHONY: all lint clean release force_release +.PHONY: all clean release force_release all: @awk -F: '/^[[:alpha:]].*:/ && !/^all:/ {print $$1}' Makefile -lint: - SKIP=black-format pre-commit run --all-files --hook-stage manual - clean: rm -rf build/ dist/ .eggs/ .tox/ diff --git a/README.md b/README.md index 0e020a5fe..c5a069bfe 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ probably the skills to scratch that itch of mine: implement `git` in a way that If you like the idea and want to learn more, please head over to [gitoxide](https://github.com/Byron/gitoxide), an implementation of 'git' in [Rust](https://www.rust-lang.org). +*(Please note that `gitoxide` is not currently available for use in Python, and that Rust is required.)* + ## GitPython GitPython is a python library used to interact with git repositories, high-level like git-porcelain, @@ -37,14 +39,14 @@ The project is open to contributions of all kinds, as well as new maintainers. ### REQUIREMENTS -GitPython needs the `git` executable to be installed on the system and available in your `PATH` for most operations. -If it is not in your `PATH`, you can help GitPython find it by setting -the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. +GitPython needs the `git` executable to be installed on the system and available in your +`PATH` for most operations. If it is not in your `PATH`, you can help GitPython find it +by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. - Git (1.7.x or newer) - Python >= 3.7 -The list of dependencies are listed in `./requirements.txt` and `./test-requirements.txt`. +The list of dependencies are listed in [`./requirements.txt`](https://github.com/gitpython-developers/GitPython/blob/main/requirements.txt) and [`./test-requirements.txt`](https://github.com/gitpython-developers/GitPython/blob/main/test-requirements.txt). The installer takes care of installing them for you. ### INSTALL @@ -55,7 +57,7 @@ GitPython and its required package dependencies can be installed in any of the f To obtain and install a copy [from PyPI](https://pypi.org/project/GitPython/), run: -```bash +```sh pip install GitPython ``` @@ -65,7 +67,7 @@ pip install GitPython If you have downloaded the source code, run this from inside the unpacked `GitPython` directory: -```bash +```sh pip install . ``` @@ -73,7 +75,7 @@ pip install . To clone the [the GitHub repository](https://github.com/gitpython-developers/GitPython) from source to work on the code, you can do it like so: -```bash +```sh git clone https://github.com/gitpython-developers/GitPython cd GitPython ./init-tests-after-clone.sh @@ -83,7 +85,7 @@ On Windows, `./init-tests-after-clone.sh` can be run in a Git Bash shell. If you are cloning [your own fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks), then replace the above `git clone` command with one that gives the URL of your fork. Or use this [`gh`](https://cli.github.com/) command (assuming you have `gh` and your fork is called `GitPython`): -```bash +```sh gh repo clone GitPython ``` @@ -91,7 +93,7 @@ Having cloned the repo, create and activate your [virtual environment](https://d Then make an [editable install](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs): -```bash +```sh pip install -e ".[test]" ``` @@ -99,15 +101,13 @@ In the less common case that you do not want to install test dependencies, `pip #### With editable *dependencies* (not preferred, and rarely needed) -In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediatley reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython. - -If you want to do that *and* you want the versions in GitPython's git submodules to be used, then pass `-e git/ext/gitdb` and/or `-e git/ext/gitdb/gitdb/ext/smmap` to `pip install`. This can be done in any order, and in separate `pip install` commands or the same one, so long as `-e` appears before *each* path. For example, you can install GitPython, gitdb, and smmap editably in the currently active virtual environment this way: +GitPython, [gitdb](gitdb), and [smmap](smmap) live in this repository as independently packaged projects. To work on all three at once, install each one editably in the same virtual environment: -```bash -pip install -e ".[test]" -e git/ext/gitdb -e git/ext/gitdb/gitdb/ext/smmap +```sh +pip install -e smmap -e gitdb -e ".[test]" ``` -The submodules must have been cloned for that to work, but that will already be the case if you have run `./init-tests-after-clone.sh`. You can use `pip list` to check which packages are installed editably and which are installed normally. +You can use `pip list` to check which packages are installed editably and which are installed normally. To reiterate, this approach should only rarely be used. For most development it is preferable to allow the gitdb and smmap dependencices to be retrieved automatically from PyPI in their latest stable packaged versions. @@ -139,42 +139,38 @@ you will encounter test failures. Ensure testing libraries are installed. This is taken care of already if you installed with: -```bash +```sh pip install -e ".[test]" ``` -Otherwise, you can run: - -```bash -pip install -r test-requirements.txt -``` +If you had installed with a command like `pip install -e .` instead, you can still run +the above command to add the testing dependencies. #### Test commands To test, run: -```bash +```sh pytest ``` -To lint, and apply automatic code formatting, run: +To lint, and apply some linting fixes as well as automatic code formatting, run: -```bash +```sh pre-commit run --all-files ``` -- Linting without modifying code can be done with: `make lint` -- Auto-formatting without other lint checks can be done with: `black .` +This includes the linting and autoformatting done by Ruff, as well as some other checks. To typecheck, run: -```bash -mypy -p git +```sh +mypy ``` #### CI (and tox) -The same linting, and running tests on all the different supported Python versions, will be performed: +Style and formatting checks, and running tests on all the different supported Python versions, will be performed: - Upon submitting a pull request. - On each push, *if* you have a fork with GitHub Actions enabled. @@ -182,16 +178,18 @@ The same linting, and running tests on all the different supported Python versio #### Configuration files -Specific tools: +Specific tools are all configured in the [`./pyproject.toml`](https://github.com/gitpython-developers/GitPython/blob/main/pyproject.toml) file: -- Configurations for `mypy`, `pytest`, `coverage.py`, and `black` are in `./pyproject.toml`. -- Configuration for `flake8` is in the `./.flake8` file. +- `pytest` (test runner) +- `coverage.py` (code coverage) +- `ruff` (linter and formatter) +- `mypy` (type checker) Orchestration tools: -- Configuration for `pre-commit` is in the `./.pre-commit-config.yaml` file. -- Configuration for `tox` is in `./tox.ini`. -- Configuration for GitHub Actions (CI) is in files inside `./.github/workflows/`. +- Configuration for `pre-commit` is in the [`./.pre-commit-config.yaml`](https://github.com/gitpython-developers/GitPython/blob/main/.pre-commit-config.yaml) file. +- Configuration for `tox` is in [`./tox.ini`](https://github.com/gitpython-developers/GitPython/blob/main/tox.ini). +- Configuration for GitHub Actions (CI) is in files inside [`./.github/workflows/`](https://github.com/gitpython-developers/GitPython/tree/main/.github/workflows). ### Contributions @@ -212,65 +210,14 @@ Please have a look at the [contributions file][contributing]. ### How to make a new release -1. Update/verify the **version** in the `VERSION` file. -2. Update/verify that the `doc/source/changes.rst` changelog file was updated. It should include a link to the forthcoming release page: `https://github.com/gitpython-developers/GitPython/releases/tag/` +1. Update/verify the **version** in the [`VERSION`](https://github.com/gitpython-developers/GitPython/blob/main/VERSION) file. +2. Update/verify that the [`doc/source/changes.rst`](https://github.com/gitpython-developers/GitPython/blob/main/doc/source/changes.rst) changelog file was updated. It should include a link to the forthcoming release page: `https://github.com/gitpython-developers/GitPython/releases/tag/` 3. Commit everything. 4. Run `git tag -s ` to tag the version in Git. 5. _Optionally_ create and activate a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment). (Then the next step can install `build` and `twine`.) 6. Run `make release`. 7. Go to [GitHub Releases](https://github.com/gitpython-developers/GitPython/releases) and publish a new one with the recently pushed tag. Generate the changelog. -### How to verify a release (DEPRECATED) - -Note that what follows is deprecated and future releases won't be signed anymore. -More details about how it came to that can be found [in this issue](https://github.com/gitpython-developers/gitdb/issues/77). - ----- - -Please only use releases from `pypi` as you can verify the respective source -tarballs. - -This script shows how to verify the tarball was indeed created by the authors of -this project: - -```bash -curl https://files.pythonhosted.org/packages/09/bc/ae32e07e89cc25b9e5c793d19a1e5454d30a8e37d95040991160f942519e/GitPython-3.1.8-py3-none-any.whl > gitpython.whl -curl https://files.pythonhosted.org/packages/09/bc/ae32e07e89cc25b9e5c793d19a1e5454d30a8e37d95040991160f942519e/GitPython-3.1.8-py3-none-any.whl.asc > gitpython-signature.asc -gpg --verify gitpython-signature.asc gitpython.whl -``` - -which outputs - -```bash -gpg: Signature made Fr 4 Sep 10:04:50 2020 CST -gpg: using RSA key 27C50E7F590947D7273A741E85194C08421980C9 -gpg: Good signature from "Sebastian Thiel (YubiKey USB-C) " [ultimate] -gpg: aka "Sebastian Thiel (In Rust I trust) " [ultimate] -``` - -You can verify that the keyid indeed matches the release-signature key provided in this -repository by looking at the keys details: - -```bash -gpg --list-packets ./release-verification-key.asc -``` - -You can verify that the commit adding it was also signed by it using: - -```bash -git show --show-signature ./release-verification-key.asc -``` - -If you would like to trust it permanently, you can import and sign it: - -```bash -gpg --import ./release-verification-key.asc -gpg --edit-key 4C08421980C9 - -> sign -> save -``` - ### Projects using GitPython - [PyDriller](https://github.com/ishepard/pydriller) @@ -291,5 +238,8 @@ gpg --edit-key 4C08421980C9 [3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license]. +One file exclusively used for fuzz testing is subject to [a separate license, detailed here](https://github.com/gitpython-developers/GitPython/blob/main/fuzzing/README.md#license). +This file is not included in the wheel or sdist packages published by the maintainers of GitPython. + [contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md [license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE diff --git a/SECURITY.md b/SECURITY.md index cbfaafdde..0aea34845 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,5 +11,6 @@ Only the latest version of GitPython can receive security updates. If a vulnerab ## Reporting a Vulnerability -Please report private portions of a vulnerability to sebastian.thiel@icloud.com that would help to reproduce and fix it. To receive updates on progress and provide -general information to the public, you can create an issue [on the issue tracker](https://github.com/gitpython-developers/GitPython/issues). +Please report private portions of a vulnerability to . Doing so helps to receive updates and collaborate on the matter, without disclosing it publicly right away. + +Vulnerabilities in GitPython's dependencies [gitdb](https://github.com/gitpython-developers/gitdb/blob/master/SECURITY.md) or [smmap](https://github.com/gitpython-developers/smmap/blob/master/SECURITY.md), which primarily exist to support GitPython, can be reported here as well, at that same link. The affected package (`GitPython`, `gitdb`, or `smmap`) can be included in the report, if known. diff --git a/VERSION b/VERSION index 6c105e676..e5c812e68 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.41 +3.1.59 diff --git a/build-release.sh b/build-release.sh index 49c13b93a..1a8dce2c2 100755 --- a/build-release.sh +++ b/build-release.sh @@ -1,5 +1,8 @@ #!/bin/bash # +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +# # This script builds a release. If run in a venv, it auto-installs its tools. # You may want to run "make release" instead of running this script directly. diff --git a/check-version.sh b/check-version.sh index dac386e46..579cf789f 100755 --- a/check-version.sh +++ b/check-version.sh @@ -1,5 +1,8 @@ #!/bin/bash # +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +# # This script checks if we are in a consistent state to build a new release. # See the release instructions in README.md for the steps to make this pass. # You may want to run "make release" instead of running this script directly. diff --git a/doc/Makefile b/doc/Makefile index ddeadbd7e..7e0d325fe 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line. BUILDDIR = build -SPHINXOPTS = -W +SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = diff --git a/doc/requirements.txt b/doc/requirements.txt index 41a7c90f1..24472ba39 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,3 @@ -sphinx==4.3.0 +sphinx >= 7.4.7, < 8 sphinx_rtd_theme sphinx-autodoc-typehints diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 79628bee2..34e727bb2 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,17 +2,229 @@ Changelog ========= +3.1.60 +====== + +Security fixes for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-g5vv-9gxw-82hx +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-whh4-5q6c-9v3x + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.60 + +3.1.59 +====== + +Security fixes for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-5xxx-qhh7-9287 +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-3wxw-xv34-2frg +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-8mcc-hrx5-hvxc +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-284h-m62q-gf8w +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-7833-fr7j-v32q + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.59 + +3.1.58 +====== + +Security fixes for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-hh9p-6wh2-4mfc +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-9rj7-rf2p-w77r +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-4gmw-gg2m-w46p +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-wvpp-8hx9-p66j +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-jm78-9fvv-mhgr +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-hmq2-w58f-27jc + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.58 + +3.1.57 +====== + +A security fix for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-3f7w-8rr8-f37f +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-539m-9xh6-q6rr + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.57 + +3.1.56 +====== + +A security fix for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-p538-c434-8v24 + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.56 + +3.1.55 +====== + +A security fix for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-94p4-4cq8-9g67 + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.55 + +3.1.54 +====== + +A security fix for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-r9mr-m37c-5fr3 +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-6p8h-3wgx-97gf +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-fjr4-x663-mwxc + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.54 + +3.1.53 +====== + +A security fix for +https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-3rp5-jjmw-4wv2 + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.53 + +3.1.52 +====== + +A security fix for +https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-rwj8-pgh3-r573. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.52 + +3.1.51 +====== + +This is primarily a security release. It prevents additional argument-injection +paths that could allow execution of arbitrary commands or writing to arbitrary +files through unsafe Git options (GHSA-956x-8gvw-wg5v), and closes bypasses of +the existing protections using abbreviated long options or joined short options +(GHSA-2f96-g7mh-g2hx). + +The release also improves support for relative worktree paths and diffs against +the empty tree, preserves stderr from failed diff processes, supports relative +configuration include paths, and includes assorted documentation, typing, test, +and CI improvements. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.51 + +3.1.50 +====== + +Save setting of configuration values, this time sections as well, as follow-up to 3.1.49. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.50 + +3.1.49 +====== + +Save setting of configuration values, +which could be used to inject other more configuration. + +Also more conforming `rev-parse` implementation. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.49 + +3.1.48 +====== + +Safe reference creation in the face of untrusted input. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.48 + + +3.1.47 +====== + +Address various security issues related to bypassing injection-protection +of unsafe Git flags. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.47 + +3.1.46 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.46 + +3.1.45 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.45 + +3.1.44 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.44 + +3.1.43 +====== + +A major visible change will be the added deprecation- or user-warnings, +and greatly improved typing. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.43 + +3.1.42 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.42 + 3.1.41 ====== -This release is relevant for security as it fixes a possible arbitary +This release is relevant for security as it fixes a possible arbitrary code execution on Windows. See this PR for details: https://github.com/gitpython-developers/GitPython/pull/1792 An advisory is available soon at: https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-2mqj-m65w-jghx See the following for all changes. -https://github.com/gitpython-developers/GitPython/releases/tag/3.1.40 +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.41 3.1.40 ====== @@ -84,7 +296,7 @@ https://github.com/gitpython-developers/gitpython/milestone/61?closed=1 but a necessary fix for https://github.com/gitpython-developers/GitPython/issues/1515. Please take a look at the PR for more information and how to bypass these protections in case they cause breakage: https://github.com/gitpython-developers/GitPython/pull/1521. - + See the following for all changes. https://github.com/gitpython-developers/gitpython/milestone/60?closed=1 @@ -149,38 +361,38 @@ https://github.com/gitpython-developers/gitpython/milestone/53?closed=1 * General: - Remove python 3.6 support - + - Remove distutils ahead of deprecation in standard library. - + - Update sphinx to 4.1.12 and use autodoc-typehints. - + - Include README as long_description on PyPI - + - Test against earliest and latest minor version available on Github Actions (e.g. 3.9.0 and 3.9.7) - + * Typing: - Add types to ALL functions. - + - Ensure py.typed is collected. - + - Increase mypy strictness with disallow_untyped_defs, warn_redundant_casts, warn_unreachable. - + - Use typing.NamedTuple and typing.OrderedDict now 3.6 dropped. - + - Make Protocol classes ABCs at runtime due to new behaviour/bug in 3.9.7 & 3.10.0-rc1 - + - Remove use of typing.TypeGuard until later release, to allow dependent libs time to update. - + - Tracking issue: https://github.com/gitpython-developers/GitPython/issues/1095 * Runtime improvements: - Add clone_multi_options support to submodule.add() - + - Delay calling get_user_id() unless essential, to support sand-boxed environments. - + - Add timeout to handle_process_output(), in case thread.join() hangs. See the following for details: diff --git a/doc/source/conf.py b/doc/source/conf.py index 9c22ca06a..809762483 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -3,29 +3,28 @@ # # This file is execfile()d with the current directory set to its containing dir. # -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). +# The contents of this file are pickled, so don't put values in the namespace that +# aren't pickleable (module imports are okay, they're removed automatically). # -# Note that not all possible configuration values are present in this -# autogenerated file. +# Note that not all possible configuration values are present in this autogenerated +# file. # -# All configuration values have a default; values that are commented out -# serve to show the default. +# All configuration values have a default; values that are commented out serve to show +# the default. -import sys import os +import sys -# If your extensions are in another directory, add it here. If the directory -# is relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. +# If your extensions are in another directory, add it here. If the directory is relative +# to the documentation root, use os.path.abspath to make it absolute, like shown here. # sys.path.append(os.path.abspath('.')) sys.path.insert(0, os.path.abspath("../..")) # General configuration # --------------------- -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# Add any Sphinx extension module names here, as strings. They can be extensions coming +# with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ["sphinx.ext.autodoc", "sphinx.ext.doctest"] # Add any paths that contain templates here, relative to this directory. @@ -44,9 +43,8 @@ project = "GitPython" copyright = "Copyright (C) 2008, 2009 Michael Trier and contributors, 2010-2015 Sebastian Thiel" -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. +# The version info for the project you're documenting, acts as replacement for |version| +# and |release|, also used in various other places throughout the built documents. # # The short X.Y version. with open(os.path.join(os.path.dirname(__file__), "..", "..", "VERSION")) as fd: @@ -55,8 +53,8 @@ # The full version, including alpha/beta/rc tags. release = VERSION -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. +# The language for content autogenerated by Sphinx. Refer to documentation for a list of +# supported languages. # language = None # There are two options for replacing |today|: either, you set today to some @@ -68,8 +66,8 @@ # List of documents that shouldn't be included in the build. # unused_docs = [] -# List of directories, relative to source directory, that shouldn't be searched -# for source files. +# List of directories, relative to source directory, that shouldn't be searched for +# source files. exclude_trees = ["build"] # The reST default role (used for this markup: `text`) to use for all documents. @@ -78,17 +76,19 @@ # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). +# If true, the current module name will be prepended to all description unit titles +# (such as .. function::). # add_module_names = True -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. +# If true, sectionauthor and moduleauthor directives will be shown in the output. +# They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +manpages_url = "https://git-scm.com/docs/{page}" + # Options for HTML output # ----------------------- @@ -96,40 +96,39 @@ html_theme = "sphinx_rtd_theme" html_theme_options = {} -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". +# The name for this set of Sphinx documents. +# If None, it defaults to " v documentation". # html_title = None -# A shorter title for the navigation bar. Default is the same as html_title. +# A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. +# The name of an image file (relative to this directory) to place at the top of the +# sidebar. # html_logo = None -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. +# The name of an image file (within the static path) to use as favicon of the docs. +# This file should be a Windows icon file (.ico) being 16x16 or 32x32 pixels large. # html_favicon = None -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". +# Add any paths that contain custom static files (such as style sheets) here, relative +# to this directory. They are copied after the builtin static files, so a file named +# "default.css" will overwrite the builtin "default.css". html_static_path = [] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, using the +# given strftime format. # html_last_updated_fmt = '%b %d, %Y' -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. +# If true, SmartyPants will be used to convert quotes and dashes to typographically +# correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # html_sidebars = {} -# Additional templates that should be rendered to pages, maps page names to -# template names. +# Additional templates that should be rendered to pages, maps page names to template +# names. # html_additional_pages = {} # If false, no module index is generated. @@ -144,9 +143,9 @@ # If true, the reST sources are included in the HTML build as _sources/. # html_copy_source = True -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. +# If true, an OpenSearch description file will be output, and all pages will contain a +# tag referring to it. The value of this option must be the base URL from which +# the finished HTML is served. # html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). @@ -171,8 +170,8 @@ ("index", "GitPython.tex", "GitPython Documentation", "Michael Trier", "manual"), ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. +# The name of an image file (relative to this directory) to place at the top of the +# title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, diff --git a/doc/source/index.rst b/doc/source/index.rst index 72db8ee5a..ca5229ac3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -21,4 +21,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 4f22a0942..e1075b1c9 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -97,11 +97,6 @@ and cloned using:: $ git clone https://github.com/gitpython-developers/GitPython git-python -Initialize all submodules to obtain the required dependencies with:: - - $ cd git-python - $ git submodule update --init --recursive - Finally verify the installation by running unit tests:: $ python -m unittest @@ -122,4 +117,3 @@ License Information =================== GitPython is licensed under the New BSD License. See the LICENSE file for more information. - diff --git a/doc/source/reference.rst b/doc/source/reference.rst index 68a7f0ba4..49d732ca2 100644 --- a/doc/source/reference.rst +++ b/doc/source/reference.rst @@ -3,13 +3,16 @@ API Reference ============= -Version -------- +Top-Level +--------- .. py:data:: git.__version__ Current GitPython version. +.. automodule:: git + :members: refresh + Objects.Base ------------ @@ -17,7 +20,7 @@ Objects.Base :members: :undoc-members: :special-members: - + Objects.Blob ------------ @@ -25,7 +28,7 @@ Objects.Blob :members: :undoc-members: :special-members: - + Objects.Commit -------------- @@ -33,7 +36,7 @@ Objects.Commit :members: :undoc-members: :special-members: - + Objects.Tag ----------- @@ -73,7 +76,7 @@ Objects.Submodule.root :members: :undoc-members: :special-members: - + Objects.Submodule.util ---------------------- @@ -81,13 +84,14 @@ Objects.Submodule.util :members: :undoc-members: :special-members: - + Objects.Util ------------- .. automodule:: git.objects.util :members: :undoc-members: + :private-members: _get_intermediate_items, _list_traverse, _traverse :special-members: Index.Base @@ -105,7 +109,7 @@ Index.Functions :members: :undoc-members: :special-members: - + Index.Types ----------- @@ -113,7 +117,7 @@ Index.Types :members: :undoc-members: :special-members: - + Index.Util ------------- @@ -121,7 +125,7 @@ Index.Util :members: :undoc-members: :special-members: - + GitCmd ------ @@ -137,7 +141,7 @@ Config :members: :undoc-members: :special-members: - + Diff ---- @@ -154,7 +158,7 @@ Exceptions :undoc-members: :special-members: - + Refs.symbolic ------------- @@ -162,7 +166,7 @@ Refs.symbolic :members: :undoc-members: :special-members: - + Refs.reference -------------- @@ -178,7 +182,7 @@ Refs.head :members: :undoc-members: :special-members: - + Refs.tag ------------ @@ -186,7 +190,7 @@ Refs.tag :members: :undoc-members: :special-members: - + Refs.remote ------------ @@ -194,7 +198,7 @@ Refs.remote :members: :undoc-members: :special-members: - + Refs.log ------------ @@ -202,7 +206,7 @@ Refs.log :members: :undoc-members: :special-members: - + Remote ------ @@ -218,7 +222,7 @@ Repo.Base :members: :undoc-members: :special-members: - + Repo.Functions -------------- @@ -227,6 +231,30 @@ Repo.Functions :undoc-members: :special-members: +Compat +------ + +.. automodule:: git.compat + :members: + :undoc-members: + :special-members: + +DB +-- + +.. automodule:: git.db + :members: + :undoc-members: + :special-members: + +Types +----- + +.. automodule:: git.types + :members: + :undoc-members: + :special-members: + Util ---- diff --git a/doc/source/roadmap.rst b/doc/source/roadmap.rst index a573df33a..34c953626 100644 --- a/doc/source/roadmap.rst +++ b/doc/source/roadmap.rst @@ -6,4 +6,3 @@ The full list of milestones including associated tasks can be found on GitHub: https://github.com/gitpython-developers/GitPython/issues Select the respective milestone to filter the list of issues accordingly. - diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index fd3b14c57..d095d3be3 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -513,6 +513,12 @@ The GitDB is a pure-python implementation of the git object database. It is the repo = Repo("path/to/repo", odbt=GitDB) +.. warning:: + ``GitDB`` may fail or become extremely slow when traversing trees in + repositories with very large commits (thousands of changed files in a + single commit). If you encounter ``RecursionError`` or excessive + slowness during tree traversal, switch to ``GitCmdObjectDB`` instead. + GitCmdObjectDB ============== diff --git a/fuzzing/LICENSE-APACHE b/fuzzing/LICENSE-APACHE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/fuzzing/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/fuzzing/LICENSE-BSD b/fuzzing/LICENSE-BSD new file mode 120000 index 000000000..ea5b60640 --- /dev/null +++ b/fuzzing/LICENSE-BSD @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/fuzzing/README.md b/fuzzing/README.md new file mode 100644 index 000000000..286f529eb --- /dev/null +++ b/fuzzing/README.md @@ -0,0 +1,226 @@ +# Fuzzing GitPython + +[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/gitpython.svg)][oss-fuzz-issue-tracker] + +This directory contains files related to GitPython's suite of fuzz tests that are executed daily on automated +infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working +with fuzzing in GitPython. + +The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is available +on [the Open Source Fuzzing Introspection website](https://introspector.oss-fuzz.com/project-profile?project=gitpython). + +## How to Contribute + +There are many ways to contribute to GitPython's fuzzing efforts! Contributions are welcomed through issues, +discussions, or pull requests on this repository. + +Areas that are particularly appreciated include: + +- **Tackling the existing backlog of open issues**. While fuzzing is an effective way to identify bugs, that information + isn't useful unless they are fixed. If you are not sure where to start, the issues tab is a great place to get ideas! +- **Improvements to this (or other) documentation** make it easier for new contributors to get involved, so even small + improvements can have a large impact over time. If you see something that could be made easier by a documentation + update of any size, please consider suggesting it! + +For everything else, such as expanding test coverage, optimizing test performance, or enhancing error detection +capabilities, jump into the "Getting Started" section below. + +## Getting Started with Fuzzing GitPython + +> [!TIP] +> **New to fuzzing or unfamiliar with OSS-Fuzz?** +> +> These resources are an excellent place to start: +> +> - [OSS-Fuzz documentation][oss-fuzz-docs] - Continuous fuzzing service for open source software. +> - [Google/fuzzing][google-fuzzing-repo] - Tutorials, examples, discussions, research proposals, and other resources + related to fuzzing. +> - [CNCF Fuzzing Handbook](https://github.com/cncf/tag-security/blob/main/security-fuzzing-handbook/handbook-fuzzing.pdf) - + A comprehensive guide for fuzzing open source software. +> - [Efficient Fuzzing Guide by The Chromium Project](https://chromium.googlesource.com/chromium/src/+/main/testing/libfuzzer/efficient_fuzzing.md) - + Explores strategies to enhance the effectiveness of your fuzz tests, recommended for those looking to optimize their + testing efforts. + +### Setting Up Your Local Environment + +Before contributing to fuzzing efforts, ensure Python and Docker are installed on your machine. Docker is required for +running fuzzers in containers provided by OSS-Fuzz and for safely executing test files directly. [Install Docker](https://docs.docker.com/get-docker/) following the official guide if you do not already have it. + +### Understanding Existing Fuzz Targets + +Review the `fuzz-targets/` directory to familiarize yourself with how existing tests are implemented. See +the [Files & Directories Overview](#files--directories-overview) for more details on the directory structure. + +### Contributing to Fuzz Tests + +Start by reviewing the [Atheris documentation][atheris-repo] and the section +on [Running Fuzzers Locally](#running-fuzzers-locally) to begin writing or improving fuzz tests. + +## Files & Directories Overview + +The `fuzzing/` directory is organized into three key areas: + +### Fuzz Targets (`fuzz-targets/`) + +Contains Python files for each fuzz test. + +**Things to Know**: + +- Each fuzz test targets a specific part of GitPython's functionality. +- Test files adhere to the naming convention: `fuzz_.py`, where `` indicates the + functionality targeted by the test. +- Any functionality that involves performing operations on input data is a possible candidate for fuzz testing, but + features that involve processing untrusted user input or parsing operations are typically going to be the most + interesting. +- The goal of these tests is to identify previously unknown or unexpected error cases caused by a given input. For that + reason, fuzz tests should gracefully handle anticipated exception cases with a `try`/`except` block to avoid false + positives that halt the fuzzing engine. + +### OSS-Fuzz Scripts (`oss-fuzz-scripts/`) + +Includes scripts for building and integrating fuzz targets with OSS-Fuzz: + +- **`container-environment-bootstrap.sh`** - Sets up the execution environment. It is responsible for fetching default + dictionary entries and ensuring all required build dependencies are installed and up-to-date. +- **`build.sh`** - Executed within the Docker container, this script builds fuzz targets with necessary instrumentation + and prepares seed corpora and dictionaries for use. + +**Where to learn more:** + +- [OSS-Fuzz documentation on the build.sh](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#buildsh) +- [See GitPython's build.sh and Dockerfile in the OSS-Fuzz repository](https://github.com/google/oss-fuzz/tree/master/projects/gitpython) + +### Local Development Helpers (`local-dev-helpers/`) + +Contains tools to make local development tasks easier. +See [the "Running Fuzzers Locally" section below](#running-fuzzers-locally) for further documentation and use cases related to files found here. + +## Running Fuzzers Locally + +> [!WARNING] +> **Some fuzz targets in this repository write to the filesystem** during execution. +> For that reason, it is strongly recommended to **always use Docker when executing fuzz targets**, even when it may be +> possible to do so without it. +> +> Although [I/O operations such as writing to disk are not considered best practice](https://github.com/google/fuzzing/blob/master/docs/good-fuzz-target.md#io), the current implementation of at least one test requires it. +> See [the "Setting Up Your Local Environment" section above](#setting-up-your-local-environment) if you do not already have Docker installed on your machine. +> +> PRs that replace disk I/O with in-memory alternatives are very much welcomed! + +### Direct Execution of Fuzz Targets + +Directly executing fuzz targets allows for quick iteration and testing of changes which can be helpful during early +development of new fuzz targets or for validating changes made to an existing test. +The [Dockerfile](./local-dev-helpers/Dockerfile) located in the `local-dev-helpers/` subdirectory provides a lightweight +container environment preconfigured with [Atheris][atheris-repo] that makes it easy to execute a fuzz target directly. + +**From the root directory of your GitPython repository clone**: + +1. Build the local development helper image: + +```shell +docker build -f fuzzing/local-dev-helpers/Dockerfile -t gitpython-fuzzdev . +``` + +2. Then execute a fuzz target inside the image, for example: + +```shell + docker run -it -v "$PWD":/src gitpython-fuzzdev python fuzzing/fuzz-targets/fuzz_config.py -atheris_runs=10000 +``` + +The above command executes [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) and exits after `10000` runs, or earlier if +the fuzzer finds an error. + +Docker CLI's `-v` flag specifies a volume mount in Docker that maps the directory in which the command is run (which +should be the root directory of your local GitPython clone) to a directory inside the container, so any modifications +made between invocations will be reflected immediately without the need to rebuild the image each time. + +### Running OSS-Fuzz Locally + +This approach uses Docker images provided by OSS-Fuzz for building and running fuzz tests locally. It offers +comprehensive features but requires a local clone of the OSS-Fuzz repository and sufficient disk space for Docker +containers. + +#### Build the Execution Environment + +Clone the OSS-Fuzz repository and prepare the Docker environment: + +```shell +git clone --depth 1 https://github.com/google/oss-fuzz.git oss-fuzz +cd oss-fuzz +python infra/helper.py build_image gitpython +python infra/helper.py build_fuzzers --sanitizer address gitpython +``` + +> [!TIP] +> The `build_fuzzers` command above accepts a local file path pointing to your GitPython repository clone as the last +> argument. +> This makes it easy to build fuzz targets you are developing locally in this repository without changing anything in +> the OSS-Fuzz repo! +> For example, if you have cloned this repository (or a fork of it) into: `~/code/GitPython` +> Then running this command would build new or modified fuzz targets using the `~/code/GitPython/fuzzing/fuzz-targets` +> directory: +> ```shell +> python infra/helper.py build_fuzzers --sanitizer address gitpython ~/code/GitPython +> ``` + +Verify the build of your fuzzers with the optional `check_build` command: + +```shell +python infra/helper.py check_build gitpython +``` + +#### Run a Fuzz Target + +Setting an environment variable for the fuzz target argument of the execution command makes it easier to quickly select +a different target between runs: + +```shell +# specify the fuzz target without the .py extension: +export FUZZ_TARGET=fuzz_config +``` + +Execute the desired fuzz target: + +```shell +python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET -- -max_total_time=60 -print_final_stats=1 +``` + +> [!TIP] +> In the example above, the "`-- -max_total_time=60 -print_final_stats=1`" portion of the command is optional but quite +> useful. +> +> Every argument provided after "`--`" in the above command is passed to the fuzzing engine directly. In this case: +> - `-max_total_time=60` tells the LibFuzzer to stop execution after 60 seconds have elapsed. +> - `-print_final_stats=1` tells the LibFuzzer to print a summary of useful metrics about the target run upon + completion. +> +> But almost any [LibFuzzer option listed in the documentation](https://llvm.org/docs/LibFuzzer.html#options) should +> work as well. + +#### Next Steps + +For detailed instructions on advanced features like reproducing OSS-Fuzz issues or using the Fuzz Introspector, refer +to [the official OSS-Fuzz documentation][oss-fuzz-docs]. + +## LICENSE + +All files located within the `fuzzing/` directory are subject to [the same license](../LICENSE) +as [the other files in this repository](../README.md#license) with one exception: + +[`fuzz_config.py`](./fuzz-targets/fuzz_config.py) was migrated to this repository from the OSS-Fuzz project's repository +where it was originally created. As such, [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) retains its original license +and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google LLC respectively) as in a header +comment, followed by a notice stating that it has have been modified contributors to GitPython. +[LICENSE-APACHE](./LICENSE-APACHE) contains the original license used by the OSS-Fuzz project repository at the time the +file was migrated. + +[oss-fuzz-repo]: https://github.com/google/oss-fuzz + +[oss-fuzz-docs]: https://google.github.io/oss-fuzz + +[oss-fuzz-issue-tracker]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:gitpython + +[google-fuzzing-repo]: https://github.com/google/fuzzing + +[atheris-repo]: https://github.com/google/atheris diff --git a/fuzzing/fuzz-targets/fuzz_blob.py b/fuzzing/fuzz-targets/fuzz_blob.py new file mode 100644 index 000000000..ce888e85f --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_blob.py @@ -0,0 +1,40 @@ +import atheris +import sys +import os +import tempfile + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + import git + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = git.Repo.init(path=temp_dir) + binsha = fdp.ConsumeBytes(20) + mode = fdp.ConsumeInt(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())) + path = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()) + + try: + blob = git.Blob(repo, binsha, mode, path) + except AssertionError as e: + if "Require 20 byte binary sha, got" in str(e): + return -1 + else: + raise e + + _ = blob.mime_type + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py new file mode 100644 index 000000000..4eddc32ff --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -0,0 +1,57 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################### +# Note: This file has been modified by contributors to GitPython. +# The original state of this file may be referenced here: +# https://github.com/google/oss-fuzz/commit/f26f254558fc48f3c9bc130b10507386b94522da +############################################################################### +import atheris +import sys +import io +import os +from configparser import MissingSectionHeaderError, ParsingError + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + import git + + +def TestOneInput(data): + sio = io.BytesIO(data) + sio.name = "/tmp/fuzzconfig.config" + git_config = git.GitConfigParser(sio) + try: + git_config.read() + except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError): + return -1 # Reject inputs raising expected exceptions + except ValueError as e: + if "embedded null byte" in str(e): + # The `os.path.expanduser` function, which does not accept strings + # containing null bytes might raise this. + return -1 + else: + raise e # Raise unanticipated exceptions as they might be bugs + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py new file mode 100644 index 000000000..d4bd68b57 --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_diff.py @@ -0,0 +1,86 @@ +import sys +import os +import io +import tempfile +from binascii import Error as BinasciiError + +import atheris + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + from git import Repo, Diff + + +class BytesProcessAdapter: + """Allows bytes to be used as process objects returned by subprocess.Popen.""" + + @atheris.instrument_func + def __init__(self, input_string): + self.stdout = io.BytesIO(input_string) + self.stderr = io.BytesIO() + + @atheris.instrument_func + def wait(self): + return 0 + + poll = wait + + +@atheris.instrument_func +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = Repo.init(path=temp_dir) + try: + diff = Diff( + repo, + a_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + b_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + a_blob_id=fdp.ConsumeBytes(20), + b_blob_id=fdp.ConsumeBytes(20), + a_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + b_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + new_file=fdp.ConsumeBool(), + deleted_file=fdp.ConsumeBool(), + copied_file=fdp.ConsumeBool(), + raw_rename_from=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + raw_rename_to=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + diff=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + change_type=fdp.PickValueInList(["A", "D", "C", "M", "R", "T", "U"]), + score=fdp.ConsumeIntInRange(0, fdp.remaining_bytes()), + ) + except BinasciiError: + return -1 + except AssertionError as e: + if "Require 20 byte binary sha, got" in str(e): + return -1 + else: + raise e + + _ = diff.__str__() + _ = diff.a_path + _ = diff.b_path + _ = diff.rename_from + _ = diff.rename_to + _ = diff.renamed_file + + diff_index = diff._index_from_patch_format( + repo, proc=BytesProcessAdapter(fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes()))) + ) + + diff._handle_diff_line( + lines_bytes=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), repo=repo, index=diff_index + ) + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_repo.py b/fuzzing/fuzz-targets/fuzz_repo.py new file mode 100644 index 000000000..7bd82c120 --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_repo.py @@ -0,0 +1,47 @@ +import atheris +import io +import sys +import os +import tempfile + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + import git + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = git.Repo.init(path=temp_dir) + + # Generate a minimal set of files based on fuzz data to minimize I/O operations. + file_paths = [os.path.join(temp_dir, f"File{i}") for i in range(min(3, fdp.ConsumeIntInRange(1, 3)))] + for file_path in file_paths: + with open(file_path, "wb") as f: + # The chosen upperbound for count of bytes we consume by writing to these + # files is somewhat arbitrary and may be worth experimenting with if the + # fuzzer coverage plateaus. + f.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) + + repo.index.add(file_paths) + repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 80))) + + fuzz_tree = git.Tree(repo, git.Tree.NULL_BIN_SHA, 0, "") + + try: + fuzz_tree._deserialize(io.BytesIO(data)) + except IndexError: + return -1 + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py new file mode 100644 index 000000000..afa653d0d --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -0,0 +1,125 @@ +import atheris +import sys +import os +import tempfile +from configparser import ParsingError +from utils import ( + setup_git_environment, + handle_exception, + get_max_filename_length, +) + +# Setup the Git environment +setup_git_environment() +from git import Repo, GitCommandError, InvalidGitRepositoryError + + +def sanitize_input(input_str, max_length=255): + """Sanitize and truncate inputs to avoid invalid Git operations.""" + sanitized = "".join(ch for ch in input_str if ch.isalnum() or ch in ("-", "_", ".")) + return sanitized[:max_length] + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as repo_temp_dir: + repo = Repo.init(path=repo_temp_dir) + repo.index.commit("Initial commit") + + try: + with tempfile.TemporaryDirectory() as submodule_temp_dir: + sub_repo = Repo.init(submodule_temp_dir, bare=fdp.ConsumeBool()) + commit_message = sanitize_input(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))) + sub_repo.index.commit(commit_message) + + submodule_name = sanitize_input( + fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, get_max_filename_length(repo.working_tree_dir)) + ) + ) + + submodule_path = os.path.relpath( + os.path.join(repo.working_tree_dir, submodule_name), + start=repo.working_tree_dir, + ) + + # Ensure submodule_path is valid + if not submodule_name or submodule_name.startswith("/") or ".." in submodule_name: + return -1 # Reject invalid input so they are not added to the corpus + + submodule = repo.create_submodule(submodule_name, submodule_path, url=sub_repo.git_dir) + repo.index.commit("Added submodule") + + with submodule.config_writer() as writer: + key_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) + value_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) + + writer.set_value( + sanitize_input(fdp.ConsumeUnicodeNoSurrogates(key_length)), + sanitize_input(fdp.ConsumeUnicodeNoSurrogates(value_length)), + ) + writer.release() + + submodule.update( + init=fdp.ConsumeBool(), + dry_run=fdp.ConsumeBool(), + force=fdp.ConsumeBool(), + ) + + submodule_repo = submodule.module() + + new_file_name = sanitize_input( + fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, get_max_filename_length(submodule_repo.working_tree_dir)) + ) + ) + new_file_path = os.path.join(submodule_repo.working_tree_dir, new_file_name) + with open(new_file_path, "wb") as new_file: + new_file.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) + + submodule_repo.index.add([new_file_path]) + submodule_repo.index.commit("Added new file to submodule") + + repo.submodule_update(recursive=fdp.ConsumeBool()) + submodule_repo.head.reset( + commit="HEAD~1", + working_tree=fdp.ConsumeBool(), + head=fdp.ConsumeBool(), + ) + + module_option_value, configuration_option_value = fdp.PickValueInList( + [(True, False), (False, True), (True, True)] + ) + submodule.remove( + module=module_option_value, + configuration=configuration_option_value, + dry_run=fdp.ConsumeBool(), + force=fdp.ConsumeBool(), + ) + repo.index.commit(f"Removed submodule {submodule_name}") + + except ( + ParsingError, + GitCommandError, + InvalidGitRepositoryError, + FileNotFoundError, + FileExistsError, + IsADirectoryError, + NotADirectoryError, + BrokenPipeError, + PermissionError, + ): + return -1 + except Exception as e: + return handle_exception(e) + + +def main(): + atheris.instrument_all() + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py new file mode 100644 index 000000000..97e6eab98 --- /dev/null +++ b/fuzzing/fuzz-targets/utils.py @@ -0,0 +1,122 @@ +import atheris # pragma: no cover +import os # pragma: no cover +import re # pragma: no cover +import traceback # pragma: no cover +import sys # pragma: no cover +from typing import Set, Tuple, List # pragma: no cover + + +@atheris.instrument_func +def is_expected_exception_message(exception: Exception, error_message_list: List[str]) -> bool: # pragma: no cover + """ + Checks if the message of a given exception matches any of the expected error messages, case-insensitively. + + Args: + exception (Exception): The exception object raised during execution. + error_message_list (List[str]): A list of error message substrings to check against the exception's message. + + Returns: + bool: True if the exception's message contains any of the substrings from the error_message_list, + case-insensitively, otherwise False. + """ + exception_message = str(exception).lower() + for error in error_message_list: + if error.lower() in exception_message: + return True + return False + + +@atheris.instrument_func +def get_max_filename_length(path: str) -> int: # pragma: no cover + """ + Get the maximum filename length for the filesystem containing the given path. + + Args: + path (str): The path to check the filesystem for. + + Returns: + int: The maximum filename length. + """ + return os.pathconf(path, "PC_NAME_MAX") + + +@atheris.instrument_func +def read_lines_from_file(file_path: str) -> list: + """Read lines from a file and return them as a list.""" + try: + with open(file_path, "r") as f: + return [line.strip() for line in f if line.strip()] + except FileNotFoundError: + print(f"File not found: {file_path}") + return [] + except IOError as e: + print(f"Error reading file {file_path}: {e}") + return [] + + +@atheris.instrument_func +def load_exception_list(file_path: str = "explicit-exceptions-list.txt") -> Set[Tuple[str, str]]: + """Load and parse the exception list from a default or specified file.""" + try: + bundle_dir = os.path.dirname(os.path.abspath(__file__)) + full_path = os.path.join(bundle_dir, file_path) + lines = read_lines_from_file(full_path) + exception_list: Set[Tuple[str, str]] = set() + for line in lines: + match = re.match(r"(.+):(\d+):", line) + if match: + file_path: str = match.group(1).strip() + line_number: str = str(match.group(2).strip()) + exception_list.add((file_path, line_number)) + return exception_list + except Exception as e: + print(f"Error loading exception list: {e}") + return set() + + +@atheris.instrument_func +def match_exception_with_traceback(exception_list: Set[Tuple[str, str]], exc_traceback) -> bool: + """Match exception traceback with the entries in the exception list.""" + for filename, lineno, _, _ in traceback.extract_tb(exc_traceback): + for file_pattern, line_pattern in exception_list: + # Ensure filename and line_number are strings for regex matching + if re.fullmatch(file_pattern, filename) and re.fullmatch(line_pattern, str(lineno)): + return True + return False + + +@atheris.instrument_func +def check_exception_against_list(exc_traceback, exception_file: str = "explicit-exceptions-list.txt") -> bool: + """Check if the exception traceback matches any entry in the exception list.""" + exception_list = load_exception_list(exception_file) + return match_exception_with_traceback(exception_list, exc_traceback) + + +@atheris.instrument_func +def handle_exception(e: Exception) -> int: + """Encapsulate exception handling logic for reusability.""" + exc_traceback = e.__traceback__ + if check_exception_against_list(exc_traceback): + return -1 + else: + raise e + + +@atheris.instrument_func +def setup_git_environment() -> None: + """Set up the environment variables for Git.""" + bundle_dir = os.path.dirname(os.path.abspath(__file__)) + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover + bundled_git_binary_path = os.path.join(bundle_dir, "git") + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path + + if not sys.warnoptions: # pragma: no cover + # The warnings filter below can be overridden by passing the -W option + # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. + import warnings + import logging + + # Fuzzing data causes some modules to generate a large number of warnings + # which are not usually interesting and make the test output hard to read, so we ignore them. + warnings.simplefilter("ignore") + logging.getLogger().setLevel(logging.ERROR) diff --git a/fuzzing/local-dev-helpers/Dockerfile b/fuzzing/local-dev-helpers/Dockerfile new file mode 100644 index 000000000..426de05dd --- /dev/null +++ b/fuzzing/local-dev-helpers/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 + +# Use the same Python version as OSS-Fuzz to accidental incompatibilities in test code +FROM python:3.8-bookworm + +LABEL project="GitPython Fuzzing Local Dev Helper" + +WORKDIR /src + +COPY . . + +# Update package managers, install necessary packages, and cleanup unnecessary files in a single RUN to keep the image smaller. +RUN apt-get update && \ + apt-get install -y git clang && \ + python -m pip install --upgrade pip && \ + python -m pip install atheris && \ + python -m pip install -e . && \ + apt-get clean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +CMD ["bash"] diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh new file mode 100644 index 000000000..c156e872d --- /dev/null +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -0,0 +1,19 @@ +# shellcheck shell=bash +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +set -euo pipefail + +python3 -m pip install . + +find "$SRC" -maxdepth 1 \ + \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \ + -exec printf '[%s] Copying: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" {} \; \ + -exec chmod a-x {} \; \ + -exec cp {} "$OUT" \; + +# Build fuzzers in $OUT. +find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do + compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." --add-data="$SRC/explicit-exceptions-list.txt:." +done diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh new file mode 100755 index 000000000..924a3cbf3 --- /dev/null +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +set -euo pipefail + +################# +# Prerequisites # +################# + +for cmd in python3 git wget zip; do + command -v "$cmd" >/dev/null 2>&1 || { + printf '[%s] Required command %s not found, exiting.\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$cmd" >&2 + exit 1 + } +done + +############# +# Functions # +############# + +download_and_concatenate_common_dictionaries() { + # Assign the first argument as the target file where all contents will be concatenated + local target_file="$1" + + # Shift the arguments so the first argument (target_file path) is removed + # and only URLs are left for the loop below. + shift + + for url in "$@"; do + wget -qO- "$url" >>"$target_file" + # Ensure there's a newline between each file's content + echo >>"$target_file" + done +} + +create_seed_corpora_zips() { + local seed_corpora_dir="$1" + local output_zip + for dir in "$seed_corpora_dir"/*; do + if [ -d "$dir" ] && [ -n "$dir" ]; then + output_zip="$SRC/$(basename "$dir")_seed_corpus.zip" + printf '[%s] Zipping the contents of %s into %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dir" "$output_zip" + zip -jur "$output_zip" "$dir"/* + fi + done +} + +prepare_dictionaries_for_fuzz_targets() { + local dictionaries_dir="$1" + local fuzz_targets_dir="$2" + local common_base_dictionary_filename="$WORK/__base.dict" + + printf '[%s] Copying .dict files from %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dictionaries_dir" "$SRC/" + cp -v "$dictionaries_dir"/*.dict "$SRC/" + + download_and_concatenate_common_dictionaries "$common_base_dictionary_filename" \ + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict" + + find "$fuzz_targets_dir" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do + if [[ -r "$common_base_dictionary_filename" ]]; then + # Strip the `.py` extension from the filename and replace it with `.dict`. + fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict" + local output_file="$SRC/$fuzz_harness_dictionary_filename" + + printf '[%s] Appending %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$common_base_dictionary_filename" "$output_file" + if [[ -s "$output_file" ]]; then + # If a dictionary file for this fuzzer already exists and is not empty, + # we append a new line to the end of it before appending any new entries. + # + # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error + # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.) + # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32 + echo >>"$output_file" + fi + cat "$common_base_dictionary_filename" >>"$output_file" + fi + done +} + +######################## +# Main execution logic # +######################## +# Seed corpora and dictionaries are hosted in a separate repository to avoid additional bloat in this repo. +# We clone into the $WORK directory because OSS-Fuzz cleans it up after building the image, keeping the image small. +git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets" + +create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora" + +prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing" + +pushd "$SRC/gitpython/" +# Search for 'raise' and 'assert' statements in Python files within GitPython's source code and submodules, saving the +# matched file path, line number, and line content to a file named 'explicit-exceptions-list.txt'. +# This file can then be used by fuzz harnesses to check exception tracebacks and filter out explicitly raised or otherwise +# anticipated exceptions to reduce false positive test failures. + +git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- '*.py' -- ':!setup.py' -- ':!test/**' -- ':!fuzzing/**' > "$SRC/explicit-exceptions-list.txt" + +popd + + +# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. +python3 -m pip install --upgrade pip +# Upgrade to the latest versions known to work at the time the below changes were introduced: +python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' diff --git a/git/__init__.py b/git/__init__.py index c6a52ef30..1b2360e3a 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -5,39 +5,7 @@ # @PydevCodeAnalysisIgnore -__version__ = "git" - -from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING - -from gitdb.util import to_hex_sha -from git.exc import * # noqa: F403 # @NoMove @IgnorePep8 -from git.types import PathLike - -try: - from git.compat import safe_decode # @NoMove @IgnorePep8 - from git.config import GitConfigParser # @NoMove @IgnorePep8 - from git.objects import * # noqa: F403 # @NoMove @IgnorePep8 - from git.refs import * # noqa: F403 # @NoMove @IgnorePep8 - from git.diff import * # noqa: F403 # @NoMove @IgnorePep8 - from git.db import * # noqa: F403 # @NoMove @IgnorePep8 - from git.cmd import Git # @NoMove @IgnorePep8 - from git.repo import Repo # @NoMove @IgnorePep8 - from git.remote import * # noqa: F403 # @NoMove @IgnorePep8 - from git.index import * # noqa: F403 # @NoMove @IgnorePep8 - from git.util import ( # @NoMove @IgnorePep8 - LockFile, - BlockingLockFile, - Stats, - Actor, - remove_password_if_present, - rmtree, - ) -except GitError as _exc: # noqa: F405 - raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc - -# __all__ must be statically defined by py.typed support -# __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] -__all__ = [ # noqa: F405 +__all__ = [ "Actor", "AmbiguousObjectName", "BadName", @@ -52,6 +20,7 @@ "CommandError", "Commit", "Diff", + "DiffConstants", "DiffIndex", "Diffable", "FetchInfo", @@ -65,18 +34,19 @@ "HEAD", "Head", "HookExecutionError", + "INDEX", "IndexEntry", "IndexFile", "IndexObject", "InvalidDBRoot", "InvalidGitRepositoryError", - "List", + "List", # Deprecated - import this from `typing` instead. "LockFile", "NULL_TREE", "NoSuchPathError", "ODBError", "Object", - "Optional", + "Optional", # Deprecated - import this from `typing` instead. "ParseError", "PathLike", "PushInfo", @@ -90,54 +60,241 @@ "RepositoryDirtyError", "RootModule", "RootUpdateProgress", - "Sequence", + "Sequence", # Deprecated - import from `typing`, or `collections.abc` in 3.9+. "StageType", "Stats", "Submodule", "SymbolicReference", - "TYPE_CHECKING", + "TYPE_CHECKING", # Deprecated - import this from `typing` instead. "Tag", "TagObject", "TagReference", "Tree", "TreeModifier", - "Tuple", - "Union", + "Tuple", # Deprecated - import this from `typing` instead. + "Union", # Deprecated - import this from `typing` instead. "UnmergedEntriesError", "UnsafeOptionError", "UnsafeProtocolError", "UnsupportedOperation", "UpdateProgress", "WorkTreeRepositoryUnsupported", + "refresh", "remove_password_if_present", "rmtree", "safe_decode", "to_hex_sha", ] +__version__ = "git" + +from typing import Any, List, Optional, Sequence, TYPE_CHECKING, Tuple, Union + +if TYPE_CHECKING: + from types import ModuleType + +import warnings + +from gitdb.util import to_hex_sha + +from git.exc import ( + AmbiguousObjectName, + BadName, + BadObject, + BadObjectType, + CacheError, + CheckoutError, + CommandError, + GitCommandError, + GitCommandNotFound, + GitError, + HookExecutionError, + InvalidDBRoot, + InvalidGitRepositoryError, + NoSuchPathError, + ODBError, + ParseError, + RepositoryDirtyError, + UnmergedEntriesError, + UnsafeOptionError, + UnsafeProtocolError, + UnsupportedOperation, + WorkTreeRepositoryUnsupported, +) +from git.types import PathLike + +try: + from git.compat import safe_decode # @NoMove + from git.config import GitConfigParser # @NoMove + from git.objects import ( # @NoMove + Blob, + Commit, + IndexObject, + Object, + RootModule, + RootUpdateProgress, + Submodule, + TagObject, + Tree, + TreeModifier, + UpdateProgress, + ) + from git.refs import ( # @NoMove + HEAD, + Head, + RefLog, + RefLogEntry, + Reference, + RemoteReference, + SymbolicReference, + Tag, + TagReference, + ) + from git.diff import ( # @NoMove + INDEX, + NULL_TREE, + Diff, + DiffConstants, + DiffIndex, + Diffable, + ) + from git.db import GitCmdObjectDB, GitDB # @NoMove + from git.cmd import Git # @NoMove + from git.repo import Repo # @NoMove + from git.remote import FetchInfo, PushInfo, Remote, RemoteProgress # @NoMove + from git.index import ( # @NoMove + BaseIndexEntry, + BlobFilter, + CheckoutError, + IndexEntry, + IndexFile, + StageType, + # NOTE: This tells type checkers what util resolves to. We delete it, and it is + # really resolved by __getattr__, which warns. See below on what to use instead. + util, + ) + from git.util import ( # @NoMove + Actor, + BlockingLockFile, + LockFile, + Stats, + remove_password_if_present, + rmtree, + ) +except GitError as _exc: + raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc + + +def _warned_import(message: str, fullname: str) -> "ModuleType": + import importlib + + warnings.warn(message, DeprecationWarning, stacklevel=3) + return importlib.import_module(fullname) + + +def _getattr(name: str) -> Any: + # TODO: If __version__ is made dynamic and lazily fetched, put that case right here. + + if name == "util": + return _warned_import( + "The expression `git.util` and the import `from git import util` actually " + "reference git.index.util, and not the git.util module accessed in " + '`from git.util import XYZ` or `sys.modules["git.util"]`. This potentially ' + "confusing behavior is currently preserved for compatibility, but may be " + "changed in the future and should not be relied on.", + fullname="git.index.util", + ) + + for names, prefix in ( + ({"head", "log", "reference", "symbolic", "tag"}, "git.refs"), + ({"base", "fun", "typ"}, "git.index"), + ): + if name not in names: + continue + + fullname = f"{prefix}.{name}" + + return _warned_import( + f"{__name__}.{name} is a private alias of {fullname} and subject to " + f"immediate removal. Use {fullname} instead.", + fullname=fullname, + ) + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +if not TYPE_CHECKING: + # NOTE: The expression `git.util` gives git.index.util and `from git import util` + # imports git.index.util, NOT git.util. It may not be feasible to change this until + # the next major version, to avoid breaking code inadvertently relying on it. + # + # - If git.index.util *is* what you want, use (or import from) that, to avoid + # confusion. + # + # - To use the "real" git.util module, write `from git.util import ...`, or if + # necessary access it as `sys.modules["git.util"]`. + # + # Note also that `import git.util` technically imports the "real" git.util... but + # the *expression* `git.util` after doing so is still git.index.util! + # + # (This situation differs from that of other indirect-submodule imports that are + # unambiguously non-public and subject to immediate removal. Here, the public + # git.util module, though different, makes less discoverable that the expression + # `git.util` refers to a non-public attribute of the git module.) + # + # This had originally come about by a wildcard import. Now that all intended imports + # are explicit, the intuitive but potentially incompatible binding occurs due to the + # usual rules for Python submodule bindings. So for now we replace that binding with + # git.index.util, delete that, and let __getattr__ handle it and issue a warning. + # + # For the same runtime behavior, it would be enough to forgo importing util, and + # delete util as created naturally; __getattr__ would behave the same. But type + # checkers would not know what util refers to when accessed as an attribute of git. + del util + + # This is "hidden" to preserve static checking for undefined/misspelled attributes. + __getattr__ = _getattr + # { Initialize git executable path + GIT_OK = None def refresh(path: Optional[PathLike] = None) -> None: - """Convenience method for setting the git executable path.""" + """Convenience method for setting the git executable path. + + :param path: + Optional path to the Git executable. If not absolute, it is resolved + immediately, relative to the current directory. + + :note: + The `path` parameter is usually omitted and cannot be used to specify a custom + command whose location is looked up in a path search on each call. See + :meth:`Git.refresh ` for details on how to achieve this. + + :note: + This calls :meth:`Git.refresh ` and sets other global + configuration according to the effect of doing so. As such, this function should + usually be used instead of using :meth:`Git.refresh ` or + :meth:`FetchInfo.refresh ` directly. + + :note: + This function is called automatically, with no arguments, at import time. + """ global GIT_OK GIT_OK = False if not Git.refresh(path=path): return if not FetchInfo.refresh(): # noqa: F405 - return # type: ignore [unreachable] + return # type: ignore[unreachable] GIT_OK = True -# } END initialize git executable path - - -################# try: refresh() except Exception as _exc: raise ImportError("Failed to initialize: {0}".format(_exc)) from _exc -################# + +# } END initialize git executable path diff --git a/git/cmd.py b/git/cmd.py index 4413182e0..193dfd4f6 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -5,16 +5,21 @@ from __future__ import annotations -import re +__all__ = ["GitMeta", "Git"] + import contextlib import io +import itertools import logging import os +import re import signal -from subprocess import Popen, PIPE, DEVNULL import subprocess -import threading +from subprocess import DEVNULL, PIPE, Popen +import sys from textwrap import dedent +import threading +import warnings from git.compat import defenc, force_bytes, safe_decode from git.exc import ( @@ -25,7 +30,6 @@ UnsafeProtocolError, ) from git.util import ( - LazyMixin, cygpath, expand_path, is_cygwin_git, @@ -56,12 +60,16 @@ overload, ) -from git.types import PathLike, Literal, TBD +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + +from git.types import Literal, PathLike, TBD if TYPE_CHECKING: - from git.repo.base import Repo from git.diff import DiffIndex - + from git.repo.base import Repo # --------------------------------------------------------------------------------- @@ -81,10 +89,7 @@ "strip_newline_in_stdout", } -log = logging.getLogger(__name__) -log.addHandler(logging.NullHandler()) - -__all__ = ("Git",) +_logger = logging.getLogger(__name__) # ============================================================================== @@ -107,22 +112,35 @@ def handle_process_output( decode_streams: bool = True, kill_after_timeout: Union[None, float] = None, ) -> None: - """Register for notifications to learn that process output is ready to read, and + R"""Register for notifications to learn that process output is ready to read, and dispatch lines to the respective line handlers. This function returns once the finalizer returns. - :param process: :class:`subprocess.Popen` instance - :param stdout_handler: f(stdout_line_string), or None - :param stderr_handler: f(stderr_line_string), or None - :param finalizer: f(proc) - wait for proc to finish + :param process: + :class:`subprocess.Popen` instance. + + :param stdout_handler: + f(stdout_line_string), or ``None``. + + :param stderr_handler: + f(stderr_line_string), or ``None``. + + :param finalizer: + f(proc) - wait for proc to finish. + :param decode_streams: - Assume stdout/stderr streams are binary and decode them before pushing - their contents to handlers. - Set it to False if ``universal_newlines == True`` (then streams are in - text mode) or if decoding must happen later (i.e. for Diffs). + Assume stdout/stderr streams are binary and decode them before pushing their + contents to handlers. + + This defaults to ``True``. Set it to ``False`` if: + + - ``universal_newlines == True``, as then streams are in text mode, or + - decoding must happen later, such as for :class:`~git.diff.Diff`\s. + :param kill_after_timeout: - float or None, Default = None + :class:`float` or ``None``, Default = ``None`` + To specify a timeout in seconds for the git command, after which the process should be killed. """ @@ -146,9 +164,9 @@ def pump_stream( handler(line) except Exception as ex: - log.error(f"Pumping {name!r} of cmd({remove_password_if_present(cmdline)}) failed due to: {ex!r}") + _logger.error(f"Pumping {name!r} of cmd({remove_password_if_present(cmdline)}) failed due to: {ex!r}") if "I/O operation on closed file" not in str(ex): - # Only reraise if the error was not due to the stream closing + # Only reraise if the error was not due to the stream closing. raise CommandError([f"<{name}-pump>"] + remove_password_if_present(cmdline), ex) from ex finally: stream.close() @@ -159,7 +177,7 @@ def pump_stream( p_stdout = process.proc.stdout if process.proc else None p_stderr = process.proc.stderr if process.proc else None else: - process = cast(Popen, process) # type: ignore [redundant-cast] + process = cast(Popen, process) # type: ignore[redundant-cast] cmdline = getattr(process, "args", "") p_stdout = process.stdout p_stderr = process.stderr @@ -194,77 +212,85 @@ def pump_stream( ) if stderr_handler: error_str: Union[str, bytes] = ( - "error: process killed because it timed out." f" kill_after_timeout={kill_after_timeout} seconds" + f"error: process killed because it timed out. kill_after_timeout={kill_after_timeout} seconds" ) if not decode_streams and isinstance(p_stderr, BinaryIO): - # Assume stderr_handler needs binary input. + # Assume stderr_handler needs binary input. error_str = cast(str, error_str) error_str = error_str.encode() - # We ignore typing on the next line because mypy does not like - # the way we inferred that stderr takes str or bytes. - stderr_handler(error_str) # type: ignore + # We ignore typing on the next line because mypy does not like the way + # we inferred that stderr takes str or bytes. + stderr_handler(error_str) # type: ignore[arg-type] if finalizer: finalizer(process) -def _safer_popen_windows( - command: Union[str, Sequence[Any]], - *, - shell: bool = False, - env: Optional[Mapping[str, str]] = None, - **kwargs: Any, -) -> Popen: - """Call :class:`subprocess.Popen` on Windows but don't include a CWD in the search. - - This avoids an untrusted search path condition where a file like ``git.exe`` in a - malicious repository would be run when GitPython operates on the repository. The - process using GitPython may have an untrusted repository's working tree as its - current working directory. Some operations may temporarily change to that directory - before running a subprocess. In addition, while by default GitPython does not run - external commands with a shell, it can be made to do so, in which case the CWD of - the subprocess, which GitPython usually sets to a repository working tree, can - itself be searched automatically by the shell. This wrapper covers all those cases. - - :note: This currently works by setting the ``NoDefaultCurrentDirectoryInExePath`` - environment variable during subprocess creation. It also takes care of passing - Windows-specific process creation flags, but that is unrelated to path search. - - :note: The current implementation contains a race condition on :attr:`os.environ`. - GitPython isn't thread-safe, but a program using it on one thread should ideally - be able to mutate :attr:`os.environ` on another, without unpredictable results. - See comments in https://github.com/gitpython-developers/GitPython/pull/1650. - """ - # CREATE_NEW_PROCESS_GROUP is needed for some ways of killing it afterwards. See: - # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal - # https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NEW_PROCESS_GROUP - creationflags = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP - - # When using a shell, the shell is the direct subprocess, so the variable must be - # set in its environment, to affect its search behavior. (The "1" can be any value.) - if shell: - safer_env = {} if env is None else dict(env) - safer_env["NoDefaultCurrentDirectoryInExePath"] = "1" - else: - safer_env = env - - # When not using a shell, the current process does the search in a CreateProcessW - # API call, so the variable must be set in our environment. With a shell, this is - # unnecessary, in versions where https://github.com/python/cpython/issues/101283 is - # patched. If not, in the rare case the ComSpec environment variable is unset, the - # shell is searched for unsafely. Setting NoDefaultCurrentDirectoryInExePath in all - # cases, as here, is simpler and protects against that. (The "1" can be any value.) - with patch_env("NoDefaultCurrentDirectoryInExePath", "1"): - return Popen( - command, - shell=shell, - env=safer_env, - creationflags=creationflags, - **kwargs, - ) +safer_popen: Callable[..., Popen] + +if sys.platform == "win32": + + def _safer_popen_windows( + command: Union[str, Sequence[Any]], + *, + shell: bool = False, + env: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> Popen: + """Call :class:`subprocess.Popen` on Windows but don't include a CWD in the + search. + + This avoids an untrusted search path condition where a file like ``git.exe`` in + a malicious repository would be run when GitPython operates on the repository. + The process using GitPython may have an untrusted repository's working tree as + its current working directory. Some operations may temporarily change to that + directory before running a subprocess. In addition, while by default GitPython + does not run external commands with a shell, it can be made to do so, in which + case the CWD of the subprocess, which GitPython usually sets to a repository + working tree, can itself be searched automatically by the shell. This wrapper + covers all those cases. + + :note: + This currently works by setting the + :envvar:`NoDefaultCurrentDirectoryInExePath` environment variable during + subprocess creation. It also takes care of passing Windows-specific process + creation flags, but that is unrelated to path search. + :note: + The current implementation contains a race condition on :attr:`os.environ`. + GitPython isn't thread-safe, but a program using it on one thread should + ideally be able to mutate :attr:`os.environ` on another, without + unpredictable results. See comments in: + https://github.com/gitpython-developers/GitPython/pull/1650 + """ + # CREATE_NEW_PROCESS_GROUP is needed for some ways of killing it afterwards. + # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal + # https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NEW_PROCESS_GROUP + creationflags = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP + + # When using a shell, the shell is the direct subprocess, so the variable must + # be set in its environment, to affect its search behavior. + if shell: + # The original may be immutable, or the caller may reuse it. Mutate a copy. + env = {} if env is None else dict(env) + env["NoDefaultCurrentDirectoryInExePath"] = "1" # The "1" can be any value. + + # When not using a shell, the current process does the search in a + # CreateProcessW API call, so the variable must be set in our environment. With + # a shell, that's unnecessary if https://github.com/python/cpython/issues/101283 + # is patched. In Python versions where it is unpatched, in the rare case the + # ComSpec environment variable is unset, the search for the shell itself is + # unsafe. Setting NoDefaultCurrentDirectoryInExePath in all cases, as done here, + # is simpler and protects against that. (As above, the "1" can be any value.) + with patch_env("NoDefaultCurrentDirectoryInExePath", "1"): + return Popen( + command, + shell=shell, + env=env, + creationflags=creationflags, + **kwargs, + ) -if os.name == "nt": safer_popen = _safer_popen_windows else: safer_popen = Popen @@ -288,7 +314,306 @@ def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], exc ## -- End Utilities -- @} -class Git(LazyMixin): +class _AutoInterrupt: + """Process wrapper that terminates the wrapped process on finalization. + + This kills/interrupts the stored process instance once this instance goes out of + scope. It is used to prevent processes piling up in case iterators stop reading. + + All attributes are wired through to the contained process object. + + The wait method is overridden to perform automatic status code checking and possibly + raise. + """ + + __slots__ = ("proc", "args", "status") + + # If this is non-zero it will override any status code during _terminate, used + # to prevent race conditions in testing. + _status_code_if_terminate: int = 0 + + def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: + self.proc = proc + self.args = args + self.status: Union[int, None] = None + + def _terminate(self) -> None: + """Terminate the underlying process.""" + if self.proc is None: + return + + proc = self.proc + self.proc = None + if proc.stdin: + proc.stdin.close() + if proc.stdout: + proc.stdout.close() + if proc.stderr: + proc.stderr.close() + # Did the process finish already so we have a return code? + try: + if proc.poll() is not None: + self.status = self._status_code_if_terminate or proc.poll() + return + except OSError as ex: + _logger.info("Ignored error after process had died: %r", ex) + + # It can be that nothing really exists anymore... + if os is None or getattr(os, "kill", None) is None: + return + + # Try to kill it. + try: + proc.terminate() + status = proc.wait() # Ensure the process goes away. + + self.status = self._status_code_if_terminate or status + except (OSError, AttributeError) as ex: + # On interpreter shutdown (notably on Windows), parts of the stdlib used by + # subprocess can already be torn down (e.g. `subprocess._winapi` becomes None), + # which can cause AttributeError during terminate(). In that case, we prefer + # to silently ignore to avoid noisy "Exception ignored in: __del__" messages. + _logger.info("Ignored error while terminating process: %r", ex) + # END exception handling + + def __del__(self) -> None: + self._terminate() + + def __getattr__(self, attr: str) -> Any: + return getattr(self.proc, attr) + + # TODO: Bad choice to mimic `proc.wait()` but with different args. + def wait(self, stderr: Union[None, str, bytes] = b"") -> int: + """Wait for the process and return its status code. + + :param stderr: + Previously read value of stderr, in case stderr is already closed. + + :warn: + May deadlock if output or error pipes are used and not handled separately. + + :raise git.exc.GitCommandError: + If the return status is not 0. + """ + if stderr is None: + stderr_b = b"" + stderr_b = force_bytes(data=stderr, encoding="utf-8") + status: Union[int, None] + if self.proc is not None: + status = self.proc.wait() + p_stderr = self.proc.stderr + else: # Assume the underlying proc was killed earlier or never existed. + status = self.status + p_stderr = None + + def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: + if stream: + try: + return stderr_b + force_bytes(stream.read()) + except (OSError, ValueError): + return stderr_b or b"" + else: + return stderr_b or b"" + + # END status handling + + if status != 0: + errstr = read_all_from_possibly_closed_stream(p_stderr) + _logger.debug("AutoInterrupt wait stderr: %r" % (errstr,)) + raise GitCommandError(remove_password_if_present(self.args), status, errstr) + return status + + +_AutoInterrupt.__name__ = "AutoInterrupt" +_AutoInterrupt.__qualname__ = "Git.AutoInterrupt" + + +class _CatFileContentStream: + """Object representing a sized read-only stream returning the contents of + an object. + + This behaves like a stream, but counts the data read and simulates an empty stream + once our sized content region is empty. + + If not all data are read to the end of the object's lifetime, we read the rest to + ensure the underlying stream continues to work. + """ + + __slots__ = ("_stream", "_nbr", "_size") + + def __init__(self, size: int, stream: IO[bytes]) -> None: + self._stream = stream + self._size = size + self._nbr = 0 # Number of bytes read. + + # Special case: If the object is empty, has null bytes, get the final + # newline right away. + if size == 0: + stream.read(1) + # END handle empty streams + + def read(self, size: int = -1) -> bytes: + bytes_left = self._size - self._nbr + if bytes_left == 0: + return b"" + if size > -1: + # Ensure we don't try to read past our limit. + size = min(bytes_left, size) + else: + # They try to read all, make sure it's not more than what remains. + size = bytes_left + # END check early depletion + data = self._stream.read(size) + self._nbr += len(data) + + # Check for depletion, read our final byte to make the stream usable by + # others. + if self._size - self._nbr == 0: + self._stream.read(1) # final newline + # END finish reading + return data + + def readline(self, size: int = -1) -> bytes: + if self._nbr == self._size: + return b"" + + # Clamp size to lowest allowed value. + bytes_left = self._size - self._nbr + if size > -1: + size = min(bytes_left, size) + else: + size = bytes_left + # END handle size + + data = self._stream.readline(size) + self._nbr += len(data) + + # Handle final byte. + if self._size - self._nbr == 0: + self._stream.read(1) + # END finish reading + + return data + + def readlines(self, size: int = -1) -> List[bytes]: + if self._nbr == self._size: + return [] + + # Leave all additional logic to our readline method, we just check the size. + out = [] + nbr = 0 + while True: + line = self.readline() + if not line: + break + out.append(line) + if size > -1: + nbr += len(line) + if nbr > size: + break + # END handle size constraint + # END readline loop + return out + + # skipcq: PYL-E0301 + def __iter__(self) -> "Git.CatFileContentStream": + return self + + def __next__(self) -> bytes: + line = self.readline() + if not line: + raise StopIteration + + return line + + next = __next__ + + def __del__(self) -> None: + bytes_left = self._size - self._nbr + if bytes_left: + # Read and discard - seeking is impossible within a stream. + # This includes any terminating newline. + self._stream.read(bytes_left + 1) + # END handle incomplete read + + +_CatFileContentStream.__name__ = "CatFileContentStream" +_CatFileContentStream.__qualname__ = "Git.CatFileContentStream" + + +_USE_SHELL_DEFAULT_MESSAGE = ( + "Git.USE_SHELL is deprecated, because only its default value of False is safe. " + "It will be removed in a future release." +) + +_USE_SHELL_DANGER_MESSAGE = ( + "Setting Git.USE_SHELL to True is unsafe and insecure, as the effect of special " + "shell syntax cannot usually be accounted for. This can result in a command " + "injection vulnerability and arbitrary code execution. Git.USE_SHELL is deprecated " + "and will be removed in a future release." +) + + +def _warn_use_shell(*, extra_danger: bool) -> None: + warnings.warn( + _USE_SHELL_DANGER_MESSAGE if extra_danger else _USE_SHELL_DEFAULT_MESSAGE, + DeprecationWarning, + stacklevel=3, + ) + + +class _GitMeta(type): + """Metaclass for :class:`Git`. + + This helps issue :class:`DeprecationWarning` if :attr:`Git.USE_SHELL` is used. + """ + + def __getattribute(cls, name: str) -> Any: + if name == "USE_SHELL": + _warn_use_shell(extra_danger=False) + return super().__getattribute__(name) + + def __setattr(cls, name: str, value: Any) -> Any: + if name == "USE_SHELL": + _warn_use_shell(extra_danger=value) + super().__setattr__(name, value) + + if not TYPE_CHECKING: + # To preserve static checking for undefined/misspelled attributes while letting + # the methods' bodies be type-checked, these are defined as non-special methods, + # then bound to special names out of view of static type checkers. (The original + # names invoke name mangling (leading "__") to avoid confusion in other scopes.) + __getattribute__ = __getattribute + __setattr__ = __setattr + + +GitMeta = _GitMeta +"""Alias of :class:`Git`'s metaclass, whether it is :class:`type` or a custom metaclass. + +Whether the :class:`Git` class has the default :class:`type` as its metaclass or uses a +custom metaclass is not documented and may change at any time. This statically checkable +metaclass alias is equivalent at runtime to ``type(Git)``. This should almost never be +used. Code that benefits from it is likely to be remain brittle even if it is used. + +In view of the :class:`Git` class's intended use and :class:`Git` objects' dynamic +callable attributes representing git subcommands, it rarely makes sense to inherit from +:class:`Git` at all. Using :class:`Git` in multiple inheritance can be especially tricky +to do correctly. Attempting uses of :class:`Git` where its metaclass is relevant, such +as when a sibling class has an unrelated metaclass and a shared lower bound metaclass +might have to be introduced to solve a metaclass conflict, is not recommended. + +:note: + The correct static type of the :class:`Git` class itself, and any subclasses, is + ``Type[Git]``. (This can be written as ``type[Git]`` in Python 3.9 later.) + + :class:`GitMeta` should never be used in any annotation where ``Type[Git]`` is + intended or otherwise possible to use. This alias is truly only for very rare and + inherently precarious situations where it is necessary to deal with the metaclass + explicitly. +""" + + +class Git(metaclass=_GitMeta): """The Git class manages communication with the Git binary. It provides a convenient interface to calling the Git binary, such as in:: @@ -297,10 +622,11 @@ class Git(LazyMixin): g.init() # calls 'git init' program rval = g.ls_files() # calls 'git ls-files' program - ``Debugging`` - Set the GIT_PYTHON_TRACE environment variable print each invocation - of the command to stdout. - Set its value to 'full' to see details about the returned values. + Debugging: + + * Set the :envvar:`GIT_PYTHON_TRACE` environment variable to print each invocation + of the command to stdout. + * Set its value to ``full`` to see details about the returned values. """ __slots__ = ( @@ -308,15 +634,33 @@ class Git(LazyMixin): "cat_file_all", "cat_file_header", "_version_info", + "_version_info_token", "_git_options", "_persistent_git_options", "_environment", ) - _excluded_ = ("cat_file_all", "cat_file_header", "_version_info") + _excluded_ = ( + "cat_file_all", + "cat_file_header", + "_version_info", + "_version_info_token", + ) re_unsafe_protocol = re.compile(r"(.+)::.+") + unsafe_git_ls_remote_options = [ + # This option allows arbitrary command execution in git-ls-remote. + "--upload-pack", + "--exec", + ] + + unsafe_git_pathspec_from_file_options = [ + # Reads pathspecs from a caller-controlled file. Some commands include an + # unmatched pathspec in their error output, which can disclose the file. + "--pathspec-from-file", + ] + def __getstate__(self) -> Dict[str, Any]: return slots_to_dict(self, exclude=self._excluded_) @@ -331,38 +675,114 @@ def __setstate__(self, d: Dict[str, Any]) -> None: GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) """Enables debugging of GitPython's git commands.""" - USE_SHELL = False - """Deprecated. If set to True, a shell will be used when executing git commands. + USE_SHELL: bool = False + """Deprecated. If set to ``True``, a shell will be used when executing git commands. + + Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any GitPython + functions should be updated to use the default value of ``False`` instead. ``True`` + is unsafe unless the effect of syntax treated specially by the shell is fully + considered and accounted for, which is not possible under most circumstances. As + detailed below, it is also no longer needed, even where it had been in the past. + + It is in many if not most cases a command injection vulnerability for an application + to set :attr:`USE_SHELL` to ``True``. Any attacker who can cause a specially crafted + fragment of text to make its way into any part of any argument to any git command + (including paths, branch names, etc.) can cause the shell to read and write + arbitrary files and execute arbitrary commands. Innocent input may also accidentally + contain special shell syntax, leading to inadvertent malfunctions. + + In addition, how a value of ``True`` interacts with some aspects of GitPython's + operation is not precisely specified and may change without warning, even before + GitPython 4.0.0 when :attr:`USE_SHELL` may be removed. This includes: + + * Whether or how GitPython automatically customizes the shell environment. + + * Whether, outside of Windows (where :class:`subprocess.Popen` supports lists of + separate arguments even when ``shell=True``), this can be used with any GitPython + functionality other than direct calls to the :meth:`execute` method. + + * Whether any GitPython feature that runs git commands ever attempts to partially + sanitize data a shell may treat specially. Currently this is not done. Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console windows in graphical Windows applications. In 2.0.8 and higher, it provides no benefit, as GitPython solves that problem more robustly and safely by using the ``CREATE_NO_WINDOW`` process creation flag on Windows. - Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any GitPython - functions should be updated to use the default value of ``False`` instead. ``True`` - is unsafe unless the effect of shell expansions is fully considered and accounted - for, which is not possible under most circumstances. + Because Windows path search differs subtly based on whether a shell is used, in rare + cases changing this from ``True`` to ``False`` may keep an unusual git "executable", + such as a batch file, from being found. To fix this, set the command name or full + path in the :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable or pass the + full path to :func:`git.refresh` (or invoke the script using a ``.exe`` shim). + + Further reading: - See: - - :meth:`Git.execute` (on the ``shell`` parameter). - - https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a - - https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + * :meth:`Git.execute` (on the ``shell`` parameter). + * https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a + * https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + * https://github.com/python/cpython/issues/91558#issuecomment-1100942950 + * https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw """ _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" _refresh_env_var = "GIT_PYTHON_REFRESH" GIT_PYTHON_GIT_EXECUTABLE = None - """Provide the full path to the git executable. Otherwise it assumes git is in the path. + """Provide the full path to the git executable. Otherwise it assumes git is in the + executable search path. - Note that the git executable is actually found during the refresh step in - the top level ``__init__``. + :note: + The git executable is actually found during the refresh step in the top level + ``__init__``. It can also be changed by explicitly calling :func:`git.refresh`. """ + _refresh_token = object() # Since None would match an initial _version_info_token. + @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: - """This gets called by the refresh function (see the top level __init__).""" + """Update information about the git executable :class:`Git` objects will use. + + Called by the :func:`git.refresh` function in the top level ``__init__``. + + :param path: + Optional path to the git executable. If not absolute, it is resolved + immediately, relative to the current directory. (See note below.) + + :note: + The top-level :func:`git.refresh` should be preferred because it calls this + method and may also update other state accordingly. + + :note: + There are three different ways to specify the command that refreshing causes + to be used for git: + + 1. Pass no `path` argument and do not set the + :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable. The command + name ``git`` is used. It is looked up in a path search by the system, in + each command run (roughly similar to how git is found when running + ``git`` commands manually). This is usually the desired behavior. + + 2. Pass no `path` argument but set the :envvar:`GIT_PYTHON_GIT_EXECUTABLE` + environment variable. The command given as the value of that variable is + used. This may be a simple command or an arbitrary path. It is looked up + in each command run. Setting :envvar:`GIT_PYTHON_GIT_EXECUTABLE` to + ``git`` has the same effect as not setting it. + + 3. Pass a `path` argument. This path, if not absolute, is immediately + resolved, relative to the current directory. This resolution occurs at + the time of the refresh. When git commands are run, they are run using + that previously resolved path. If a `path` argument is passed, the + :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable is not + consulted. + + :note: + Refreshing always sets the :attr:`Git.GIT_PYTHON_GIT_EXECUTABLE` class + attribute, which can be read on the :class:`Git` class or any of its + instances to check what command is used to run git. This attribute should + not be confused with the related :envvar:`GIT_PYTHON_GIT_EXECUTABLE` + environment variable. The class attribute is set no matter how refreshing is + performed. + """ # Discern which path to refresh with. if path is not None: new_git = os.path.expanduser(path) @@ -372,10 +792,12 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: # Keep track of the old and new git executable path. old_git = cls.GIT_PYTHON_GIT_EXECUTABLE + old_refresh_token = cls._refresh_token cls.GIT_PYTHON_GIT_EXECUTABLE = new_git + cls._refresh_token = object() # Test if the new git executable path is valid. A GitCommandNotFound error is - # spawned by us. A PermissionError is spawned if the git executable cannot be + # raised by us. A PermissionError is raised if the git executable cannot be # executed for whatever reason. has_git = False try: @@ -393,7 +815,7 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: The git executable must be specified in one of the following ways: - be included in your $PATH - be set via $%s - - explicitly set via git.refresh() + - explicitly set via git.refresh() """ ) % cls._git_exec_env_var @@ -401,88 +823,84 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: # Revert to whatever the old_git was. cls.GIT_PYTHON_GIT_EXECUTABLE = old_git + cls._refresh_token = old_refresh_token if old_git is None: # On the first refresh (when GIT_PYTHON_GIT_EXECUTABLE is None) we only # are quiet, warn, or error depending on the GIT_PYTHON_REFRESH value. - # Determine what the user wants to happen during the initial refresh we + # Determine what the user wants to happen during the initial refresh. We # expect GIT_PYTHON_REFRESH to either be unset or be one of the # following values: # - # 0|q|quiet|s|silence|n|none - # 1|w|warn|warning - # 2|r|raise|e|error + # 0|q|quiet|s|silence|silent|n|none + # 1|w|warn|warning|l|log + # 2|r|raise|e|error|exception mode = os.environ.get(cls._refresh_env_var, "raise").lower() - quiet = ["quiet", "q", "silence", "s", "none", "n", "0"] - warn = ["warn", "w", "warning", "1"] - error = ["error", "e", "raise", "r", "2"] + quiet = ["quiet", "q", "silence", "s", "silent", "none", "n", "0"] + warn = ["warn", "w", "warning", "log", "l", "1"] + error = ["error", "e", "exception", "raise", "r", "2"] if mode in quiet: pass elif mode in warn or mode in error: - err = ( - dedent( - """\ + err = dedent( + """\ %s All git commands will error until this is rectified. - This initial warning can be silenced or aggravated in the future by setting the + This initial message can be silenced or aggravated in the future by setting the $%s environment variable. Use one of the following values: - - %s: for no warning or exception - - %s: for a printed warning + - %s: for no message or exception + - %s: for a warning message (logging level CRITICAL, displayed by default) - %s: for a raised exception Example: export %s=%s """ - ) - % ( - err, - cls._refresh_env_var, - "|".join(quiet), - "|".join(warn), - "|".join(error), - cls._refresh_env_var, - quiet[0], - ) + ) % ( + err, + cls._refresh_env_var, + "|".join(quiet), + "|".join(warn), + "|".join(error), + cls._refresh_env_var, + quiet[0], ) if mode in warn: - print("WARNING: %s" % err) + _logger.critical(err) else: raise ImportError(err) else: - err = ( - dedent( - """\ + err = dedent( + """\ %s environment variable has been set but it has been set with an invalid value. Use only the following values: - - %s: for no warning or exception - - %s: for a printed warning + - %s: for no message or exception + - %s: for a warning message (logging level CRITICAL, displayed by default) - %s: for a raised exception """ - ) - % ( - cls._refresh_env_var, - "|".join(quiet), - "|".join(warn), - "|".join(error), - ) + ) % ( + cls._refresh_env_var, + "|".join(quiet), + "|".join(warn), + "|".join(error), ) raise ImportError(err) - # We get here if this was the init refresh and the refresh mode was not - # error. Go ahead and set the GIT_PYTHON_GIT_EXECUTABLE such that we - # discern the difference between a first import and a second import. + # We get here if this was the initial refresh and the refresh mode was + # not error. Go ahead and set the GIT_PYTHON_GIT_EXECUTABLE such that we + # discern the difference between the first refresh at import time + # and subsequent calls to git.refresh or this refresh method. cls.GIT_PYTHON_GIT_EXECUTABLE = cls.git_exec_name else: # After the first refresh (when GIT_PYTHON_GIT_EXECUTABLE is no longer # None) we raise an exception. - raise GitCommandNotFound("git", err) + raise GitCommandNotFound(new_git, err) return has_git @@ -492,31 +910,34 @@ def is_cygwin(cls) -> bool: @overload @classmethod - def polish_url(cls, url: str, is_cygwin: Literal[False] = ...) -> str: - ... + def polish_url(cls, url: str, is_cygwin: Literal[False] = ..., expand_vars: bool = ...) -> str: ... @overload @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: - ... + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None, expand_vars: bool = True) -> str: ... @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: - """Remove any backslashes from urls to be written in config files. + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None, expand_vars: bool = True) -> PathLike: + """Remove any backslashes from URLs to be written in config files. + + Windows might create config files containing paths with backslashes, but git + stops liking them as it will escape the backslashes. Hence we undo the escaping + just to be sure. - Windows might create config files containing paths with backslashes, - but git stops liking them as it will escape the backslashes. Hence we - undo the escaping just to be sure. + :param expand_vars: + Expand environment variables and an initial ``~``. Disable this for values + obtained from an untrusted source, such as remote URLs. """ if is_cygwin is None: is_cygwin = cls.is_cygwin() if is_cygwin: - url = cygpath(url) + url = cygpath(url, expand_vars=expand_vars) else: - url = os.path.expandvars(url) - if url.startswith("~"): - url = os.path.expanduser(url) + if expand_vars: + url = os.path.expandvars(url) + if url.startswith("~"): + url = os.path.expanduser(url) url = url.replace("\\\\", "\\").replace("\\", "/") return url @@ -524,9 +945,9 @@ def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: def check_unsafe_protocols(cls, url: str) -> None: """Check for unsafe protocols. - Apart from the usual protocols (http, git, ssh), - Git allows "remote helpers" that have the form ``::
``. - One of these helpers (``ext::``) can be used to invoke any arbitrary command. + Apart from the usual protocols (http, git, ssh), Git allows "remote helpers" + that have the form ``::
``. One of these helpers (``ext::``) + can be used to invoke any arbitrary command. See: @@ -541,239 +962,126 @@ def check_unsafe_protocols(cls, url: str) -> None: ) @classmethod - def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> None: - """Check for unsafe options. + def _canonicalize_option_name(cls, option: str) -> str: + """Return the option name used for unsafe-option checks. - Some options that are passed to `git ` can be used to execute - arbitrary commands, this are blocked by default. + Examples: + ``"--upload-pack=/tmp/helper"`` -> ``"upload-pack"`` + ``"upload_pack"`` -> ``"upload-pack"`` + ``"--config core.filemode=false"`` -> ``"config"`` """ - # Options can be of the form `foo` or `--foo bar` `--foo=bar`, - # so we need to check if they start with "--foo" or if they are equal to "foo". - bare_unsafe_options = [option.lstrip("-") for option in unsafe_options] + option_name = option.lstrip("-").split("=", 1)[0] + option_tokens = option_name.split(None, 1) + if not option_tokens: + return "" + return dashify(option_tokens[0]) + + @classmethod + def check_unsafe_options( + cls, options: List[str], unsafe_options: List[str], clusterable_short_options: str = "46flnqsv" + ) -> None: + """Raise :class:`~git.exc.UnsafeOptionError` for blocked option spellings. + + In addition to exact matches, this rejects abbreviated long options accepted + by Git (for example, ``--upl`` for ``--upload-pack``) and unsafe short options + whose values are joined to the same token, including after clusterable flags + (for example, ``-uVALUE`` and ``-fuVALUE``). + + A list containing only bare names is treated as normalized keyword arguments, + so multi-character names such as ``upload_p`` are checked as long-option + abbreviations. If any item starts with ``-``, the list is treated as tokenized + command-line input: bare items can be option values and are not checked as + abbreviations. Thus ``["--origin", "upload"]`` is allowed. Single-dash options + use short-option parsing rather than broad prefix matching, preserving safe + attached values such as ``-oupstream`` and ``-bcurrent``. + + Some options passed to ``git `` can execute arbitrary commands and + are therefore blocked by default unless the caller explicitly allows them. + """ + # Options can be of the form `foo`, `--foo`, `--foo bar`, or `--foo=bar`. + # Git accepts any unambiguous prefix of a long option, so an abbreviated + # spelling such as `--upl` for `--upload-pack` must be rejected too. An + # option is unsafe if its canonical name is a prefix of any blocked + # option's canonical name. Only long options and multi-character kwargs + # can be abbreviations; single-character short options remain exact-match + # only. + canonical_unsafe_options = {cls._canonicalize_option_name(option): option for option in unsafe_options} + unsafe_short_options = { + canonical: option + for canonical, option in canonical_unsafe_options.items() + if option.startswith("-") and not option.startswith("--") and len(canonical) == 1 + } + # These value-less Git flags can be clustered before another short option + # (for example, ``-fuVALUE``). Stop at any other character because it may + # begin an attached value, as ``o`` does in the safe option ``-oupstream``. + clusterable_short_options_set = frozenset(clusterable_short_options) + options_are_kwargs = all(not option.startswith("-") for option in options) for option in options: - for unsafe_option, bare_option in zip(unsafe_options, bare_unsafe_options): - if option.startswith(unsafe_option) or option == bare_option: + candidate = cls._canonicalize_option_name(option) + if not candidate: + continue + unsafe_option = canonical_unsafe_options.get(candidate) + if unsafe_option is not None: + raise UnsafeOptionError(f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it.") + option_token = option.split("=", 1)[0].split(None, 1)[0] + if option_token.startswith("-") and not option_token.startswith("--"): + for option_char in option_token[1:]: + unsafe_option = unsafe_short_options.get(option_char) + if unsafe_option is not None: + raise UnsafeOptionError( + f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." + ) + if option_char not in clusterable_short_options_set: + break + if not (option.startswith("--") or (options_are_kwargs and len(candidate) > 1)): + continue + for canonical, unsafe_option in canonical_unsafe_options.items(): + if canonical.startswith(candidate): raise UnsafeOptionError( f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." ) - class AutoInterrupt: - """Process wrapper that terminates the wrapped process on finalization. - - This kills/interrupts the stored process instance once this instance goes out of - scope. It is used to prevent processes piling up in case iterators stop reading. - - All attributes are wired through to the contained process object. - - The wait method is overridden to perform automatic status code checking and - possibly raise. - """ - - __slots__ = ("proc", "args", "status") - - # If this is non-zero it will override any status code during - # _terminate, used to prevent race conditions in testing. - _status_code_if_terminate: int = 0 - - def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: - self.proc = proc - self.args = args - self.status: Union[int, None] = None - - def _terminate(self) -> None: - """Terminate the underlying process.""" - if self.proc is None: - return - - proc = self.proc - self.proc = None - if proc.stdin: - proc.stdin.close() - if proc.stdout: - proc.stdout.close() - if proc.stderr: - proc.stderr.close() - # Did the process finish already so we have a return code? - try: - if proc.poll() is not None: - self.status = self._status_code_if_terminate or proc.poll() - return - except OSError as ex: - log.info("Ignored error after process had died: %r", ex) - - # It can be that nothing really exists anymore... - if os is None or getattr(os, "kill", None) is None: - return - - # Try to kill it. - try: - proc.terminate() - status = proc.wait() # Ensure the process goes away. - - self.status = self._status_code_if_terminate or status - except OSError as ex: - log.info("Ignored error after process had died: %r", ex) - # END exception handling - - def __del__(self) -> None: - self._terminate() - - def __getattr__(self, attr: str) -> Any: - return getattr(self.proc, attr) - - # TODO: Bad choice to mimic `proc.wait()` but with different args. - def wait(self, stderr: Union[None, str, bytes] = b"") -> int: - """Wait for the process and return its status code. - - :param stderr: Previously read value of stderr, in case stderr is already closed. - :warn: May deadlock if output or error pipes are used and not handled separately. - :raise GitCommandError: If the return status is not 0. - """ - if stderr is None: - stderr_b = b"" - stderr_b = force_bytes(data=stderr, encoding="utf-8") - status: Union[int, None] - if self.proc is not None: - status = self.proc.wait() - p_stderr = self.proc.stderr - else: # Assume the underlying proc was killed earlier or never existed. - status = self.status - p_stderr = None - - def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: - if stream: - try: - return stderr_b + force_bytes(stream.read()) - except (OSError, ValueError): - return stderr_b or b"" - else: - return stderr_b or b"" - - # END status handling - - if status != 0: - errstr = read_all_from_possibly_closed_stream(p_stderr) - log.debug("AutoInterrupt wait stderr: %r" % (errstr,)) - raise GitCommandError(remove_password_if_present(self.args), status, errstr) - return status - - # END auto interrupt - - class CatFileContentStream: - """Object representing a sized read-only stream returning the contents of - an object. - - This behaves like a stream, but counts the data read and simulates an empty - stream once our sized content region is empty. - - If not all data are read to the end of the object's lifetime, we read the - rest to ensure the underlying stream continues to work. - """ - - __slots__: Tuple[str, ...] = ("_stream", "_nbr", "_size") - - def __init__(self, size: int, stream: IO[bytes]) -> None: - self._stream = stream - self._size = size - self._nbr = 0 # Number of bytes read. - - # Special case: If the object is empty, has null bytes, get the - # final newline right away. - if size == 0: - stream.read(1) - # END handle empty streams - - def read(self, size: int = -1) -> bytes: - bytes_left = self._size - self._nbr - if bytes_left == 0: - return b"" - if size > -1: - # Ensure we don't try to read past our limit. - size = min(bytes_left, size) - else: - # They try to read all, make sure it's not more than what remains. - size = bytes_left - # END check early depletion - data = self._stream.read(size) - self._nbr += len(data) - - # Check for depletion, read our final byte to make the stream usable by others. - if self._size - self._nbr == 0: - self._stream.read(1) # final newline - # END finish reading - return data - - def readline(self, size: int = -1) -> bytes: - if self._nbr == self._size: - return b"" - - # Clamp size to lowest allowed value. - bytes_left = self._size - self._nbr - if size > -1: - size = min(bytes_left, size) - else: - size = bytes_left - # END handle size - - data = self._stream.readline(size) - self._nbr += len(data) - - # Handle final byte. - if self._size - self._nbr == 0: - self._stream.read(1) - # END finish reading - - return data - - def readlines(self, size: int = -1) -> List[bytes]: - if self._nbr == self._size: - return [] - - # Leave all additional logic to our readline method, we just check the size. - out = [] - nbr = 0 - while True: - line = self.readline() - if not line: - break - out.append(line) - if size > -1: - nbr += len(line) - if nbr > size: - break - # END handle size constraint - # END readline loop - return out - - # skipcq: PYL-E0301 - def __iter__(self) -> "Git.CatFileContentStream": - return self - - def __next__(self) -> bytes: - line = self.readline() - if not line: - raise StopIteration - - return line + @classmethod + def _option_candidates(cls, args: Sequence[Any] = (), kwargs: Optional[Mapping[str, Any]] = None) -> List[str]: + """Collect possible option spellings before command-line transformation.""" + options = [ + option for option in cls._unpack_args([arg for arg in args if arg is not None]) if option.startswith("-") + ] + if kwargs: + split_single_char_options = kwargs.get("split_single_char_options", True) + for key, value in kwargs.items(): + values = value if isinstance(value, (list, tuple)) else (value,) + if any(value is True or (value is not False and value is not None) for value in values): + key = str(key) + if len(key) != 1: + options.append(f"--{dashify(key)}") + elif split_single_char_options: + options.append(f"-{key}") + options.extend( + str(value) + for value in values + if value is not True and value not in (False, None) and str(value).startswith("-") + ) + else: + options.extend( + f"-{key}" if value is True else f"-{key}{value}" + for value in values + if value is True or (value is not False and value is not None) + ) + return options - next = __next__ + AutoInterrupt: TypeAlias = _AutoInterrupt - def __del__(self) -> None: - bytes_left = self._size - self._nbr - if bytes_left: - # Read and discard - seeking is impossible within a stream. - # This includes any terminating newline. - self._stream.read(bytes_left + 1) - # END handle incomplete read + CatFileContentStream: TypeAlias = _CatFileContentStream - def __init__(self, working_dir: Union[None, PathLike] = None): + def __init__(self, working_dir: Union[None, PathLike] = None) -> None: """Initialize this instance with: :param working_dir: - Git directory we should work in. If None, we always work in the current - directory as returned by :func:`os.getcwd`. - It is meant to be the working tree directory if available, or the - ``.git`` directory in case of bare repositories. + Git directory we should work in. If ``None``, we always work in the current + directory as returned by :func:`os.getcwd`. + This is meant to be the working tree directory if available, or the + ``.git`` directory in case of bare repositories. """ super().__init__() self._working_dir = expand_path(working_dir) @@ -783,20 +1091,28 @@ def __init__(self, working_dir: Union[None, PathLike] = None): # Extra environment variables to pass to git commands self._environment: Dict[str, str] = {} + # Cached version slots + self._version_info: Union[Tuple[int, ...], None] = None + self._version_info_token: object = None + # Cached command slots self.cat_file_header: Union[None, TBD] = None self.cat_file_all: Union[None, TBD] = None + def __getattribute__(self, name: str) -> Any: + if name == "USE_SHELL": + _warn_use_shell(extra_danger=False) + return super().__getattribute__(name) + def __getattr__(self, name: str) -> Any: - """A convenience method as it allows to call the command as if it was - an object. + """A convenience method as it allows to call the command as if it was an object. :return: - Callable object that will execute call :meth:`_call_process` with - your arguments. + Callable object that will execute call :meth:`_call_process` with your + arguments. """ - if name[0] == "_": - return LazyMixin.__getattr__(self, name) + if name.startswith("_"): + return super().__getattribute__(name) return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) def set_persistent_git_options(self, **kwargs: Any) -> None: @@ -805,25 +1121,27 @@ def set_persistent_git_options(self, **kwargs: Any) -> None: :param kwargs: A dict of keyword arguments. - These arguments are passed as in :meth:`_call_process`, but will be - passed to the git command rather than the subcommand. + These arguments are passed as in :meth:`_call_process`, but will be passed + to the git command rather than the subcommand. """ self._persistent_git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) - def _set_cache_(self, attr: str) -> None: - if attr == "_version_info": - # We only use the first 4 numbers, as everything else could be strings in fact (on Windows). - process_version = self._call_process("version") # Should be as default *args and **kwargs used. - version_numbers = process_version.split(" ")[2] + def ls_remote( + self, + *args: Any, + allow_unsafe_options: bool = False, + **kwargs: Any, + ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: + """List references in a remote repository. - self._version_info = cast( - Tuple[int, int, int, int], - tuple(int(n) for n in version_numbers.split(".")[:4] if n.isdigit()), - ) - else: - super()._set_cache_(attr) - # END handle version info + :param allow_unsafe_options: + Allow unsafe options, like ``--upload-pack`` or ``--exec``. + """ + if not allow_unsafe_options: + candidate_options = self._option_candidates(args, kwargs) + Git.check_unsafe_options(options=candidate_options, unsafe_options=self.unsafe_git_ls_remote_options) + return self._call_process("ls_remote", *args, **kwargs) @property def working_dir(self) -> Union[None, PathLike]: @@ -831,18 +1149,40 @@ def working_dir(self) -> Union[None, PathLike]: return self._working_dir @property - def version_info(self) -> Tuple[int, int, int, int]: + def version_info(self) -> Tuple[int, ...]: """ - :return: tuple(int, int, int, int) tuple with integers representing the major, minor - and additional version numbers as parsed from git version. + :return: Tuple with integers representing the major, minor and additional + version numbers as parsed from :manpage:`git-version(1)`. Up to four fields + are used. This value is generated on demand and is cached. """ + # Refreshing is global, but version_info caching is per-instance. + refresh_token = self._refresh_token # Copy token in case of concurrent refresh. + + # Use the cached version if obtained after the most recent refresh. + if self._version_info_token is refresh_token: + assert self._version_info is not None, "Bug: corrupted token-check state" + return self._version_info + + # Run "git version" and parse it. + process_version = self._call_process("version") + version_string = process_version.split(" ")[2] + version_fields = version_string.split(".")[:4] + leading_numeric_fields = itertools.takewhile(str.isdigit, version_fields) + self._version_info = tuple(map(int, leading_numeric_fields)) + + # This value will be considered valid until the next refresh. + self._version_info_token = refresh_token return self._version_info @overload - def execute(self, command: Union[str, Sequence[Any]], *, as_process: Literal[True]) -> "AutoInterrupt": - ... + def execute( + self, + command: Union[str, Sequence[Any]], + *, + as_process: Literal[True], + ) -> "AutoInterrupt": ... @overload def execute( @@ -851,8 +1191,7 @@ def execute( *, as_process: Literal[False] = False, stdout_as_string: Literal[True], - ) -> Union[str, Tuple[int, str, str]]: - ... + ) -> Union[str, Tuple[int, str, str]]: ... @overload def execute( @@ -861,8 +1200,7 @@ def execute( *, as_process: Literal[False] = False, stdout_as_string: Literal[False] = False, - ) -> Union[bytes, Tuple[int, bytes, str]]: - ... + ) -> Union[bytes, Tuple[int, bytes, str]]: ... @overload def execute( @@ -872,8 +1210,7 @@ def execute( with_extended_output: Literal[False], as_process: Literal[False], stdout_as_string: Literal[True], - ) -> str: - ... + ) -> str: ... @overload def execute( @@ -883,8 +1220,7 @@ def execute( with_extended_output: Literal[False], as_process: Literal[False], stdout_as_string: Literal[False], - ) -> bytes: - ... + ) -> bytes: ... def execute( self, @@ -908,9 +1244,28 @@ def execute( information (stdout). :param command: - The command argument list to execute. - It should be a sequence of program arguments, or a string. The - program to execute is the first item in the args sequence or string. + The command to execute. A sequence of program arguments is recommended. + A string is also accepted, but its meaning is strongly platform-dependent. + + By default, a shell is not used. On Unix-like systems, a string is the whole + program name (so ``"git log -n 1"`` raises :class:`GitCommandNotFound`). On + Windows, the program parses the arguments itself, so multi-word strings can + work but are not portable. + + Avoid ``shell=True`` (and :attr:`Git.USE_SHELL`): this runs the command in + a shell, which is generally unsafe. The shell interprets metacharacters + such as ``;``, ``|``, ``&``, ``$(...)``, ``$VAR``, ``%VAR%``, and ``^`` + (depending on the platform) as syntax. Any untrusted text in the command + can then execute arbitrary OS commands. See :attr:`Git.USE_SHELL`. + + Producing a sequence automatically by :func:`shlex.split` and passing it + as the command is far safer than ``shell=True``. But :func:`shlex.split` + parses POSIX shell syntax on all systems, and the result is still unsafe + for anything but *fixed, fully trusted* strings. Do not use it on strings + built by interpolating values: whitespace or quoting in an untrusted value + can still inject arguments. For input derived in any way from untrusted + data, build the argument sequence yourself, while ensuring each argument + is fully sanitized. :param istream: Standard input filehandle passed to :class:`subprocess.Popen`. @@ -932,24 +1287,20 @@ def execute( :param output_stream: If set to a file-like object, data produced by the git command will be - output to the given stream directly. - This feature only has any effect if `as_process` is False. Processes will - always be created with a pipe due to issues with subprocess. - This merely is a workaround as data will be copied from the - output pipe to the given output stream directly. - Judging from the implementation, you shouldn't use this parameter! + copied to the given stream instead of being returned as a string. + This feature only has any effect if `as_process` is ``False``. :param stdout_as_string: - If False, the command's standard output will be bytes. Otherwise, it will be - decoded into a string using the default encoding (usually UTF-8). + If ``False``, the command's standard output will be bytes. Otherwise, it + will be decoded into a string using the default encoding (usually UTF-8). The latter can fail, if the output contains binary data. :param kill_after_timeout: Specifies a timeout in seconds for the git command, after which the process - should be killed. This will have no effect if `as_process` is set to True. - It is set to None by default and will let the process run until the timeout - is explicitly specified. Uses of this feature should be carefully - considered, due to the following limitations: + should be killed. This will have no effect if `as_process` is set to + ``True``. It is set to ``None`` by default and will let the process run + until the timeout is explicitly specified. Uses of this feature should be + carefully considered, due to the following limitations: 1. This feature is not supported at all on Windows. 2. Effectiveness may vary by operating system. ``ps --ppid`` is used to @@ -958,20 +1309,21 @@ def execute( 3. Deeper descendants do not receive signals, though they may sometimes terminate as a consequence of their parent processes being killed. 4. `kill_after_timeout` uses ``SIGKILL``, which can have negative side - effects on a repository. For example, stale locks in case of ``git gc`` - could render the repository incapable of accepting changes until the lock - is manually removed. + effects on a repository. For example, stale locks in case of + :manpage:`git-gc(1)` could render the repository incapable of accepting + changes until the lock is manually removed. :param with_stdout: - If True, default True, we open stdout on the created process. + If ``True``, default ``True``, we open stdout on the created process. :param universal_newlines: - if True, pipes will be opened as text, and lines are split at - all known line endings. + If ``True``, pipes will be opened as text, and lines are split at all known + line endings. :param shell: - Whether to invoke commands through a shell (see `Popen(..., shell=True)`). - If this is not `None`, it overrides :attr:`USE_SHELL`. + Whether to invoke commands through a shell + (see :class:`Popen(..., shell=True) `). + If this is not ``None``, it overrides :attr:`USE_SHELL`. Passing ``shell=True`` to this or any other GitPython function should be avoided, as it is unsafe under most circumstances. This is because it is @@ -981,13 +1333,19 @@ def execute( needed (nor useful) to work around any known operating system specific issues. + On Unix-like systems, when migrating away from passing string commands with + ``shell=True``, :func:`shlex.split` may serve as a transitional step in rare + cases, with extreme care. (Drop ``shell=True`` and pass the resulting + sequence as the command.) See the `command` parameter above on the risks. + :param env: - A dictionary of environment variables to be passed to :class:`subprocess.Popen`. + A dictionary of environment variables to be passed to + :class:`subprocess.Popen`. :param max_chunk_size: - Maximum number of bytes in one chunk of data passed to the output_stream in - one invocation of write() method. If the given number is not positive then - the default value is used. + Maximum number of bytes in one chunk of data passed to the `output_stream` + in one invocation of its ``write()`` method. If the given number is not + positive then the default value is used. :param strip_newline_in_stdout: Whether to strip the trailing ``\n`` of the command stdout. @@ -998,30 +1356,33 @@ def execute( specify may not be the same ones. :return: - * str(output) if extended_output = False (Default) - * tuple(int(status), str(stdout), str(stderr)) if extended_output = True + * str(output), if `extended_output` is ``False`` (Default) + * tuple(int(status), str(stdout), str(stderr)), + if `extended_output` is ``True`` - If output_stream is True, the stdout value will be your output stream: - * output_stream if extended_output = False - * tuple(int(status), output_stream, str(stderr)) if extended_output = True + If `output_stream` is ``True``, the stdout value will be your output stream: + + * output_stream, if `extended_output` is ``False`` + * tuple(int(status), output_stream, str(stderr)), + if `extended_output` is ``True`` Note that git is executed with ``LC_MESSAGES="C"`` to ensure consistent output regardless of system language. - :raise GitCommandError: + :raise git.exc.GitCommandError: :note: - If you add additional keyword arguments to the signature of this method, - you must update the execute_kwargs tuple housed in this module. + If you add additional keyword arguments to the signature of this method, you + must update the ``execute_kwargs`` variable housed in this module. """ # Remove password for the command if present. redacted_command = remove_password_if_present(command) if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != "full" or as_process): - log.info(" ".join(redacted_command)) + _logger.info(" ".join(redacted_command)) # Allow the user to have the command executed in their working dir. try: - cwd = self._working_dir or os.getcwd() # type: Union[None, str] + cwd = self._working_dir or os.getcwd() # type: Optional[PathLike] if not os.access(str(cwd), os.X_OK): cwd = None except FileNotFoundError: @@ -1040,21 +1401,26 @@ def execute( if inline_env is not None: env.update(inline_env) - if os.name == "nt": - cmd_not_found_exception = OSError + if sys.platform == "win32": if kill_after_timeout is not None: raise GitCommandError( redacted_command, '"kill_after_timeout" feature is not supported on Windows.', ) + cmd_not_found_exception = OSError else: cmd_not_found_exception = FileNotFoundError # END handle stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") if shell is None: - shell = self.USE_SHELL - log.debug( + # Get the value of USE_SHELL with no deprecation warning. Do this without + # warnings.catch_warnings, to avoid a race condition with application code + # configuring warnings. The value could be looked up in type(self).__dict__ + # or Git.__dict__, but those can break under some circumstances. This works + # the same as self.USE_SHELL in more situations; see Git.__getattribute__. + shell = super().__getattribute__("USE_SHELL") + _logger.debug( "Popen(%s, cwd=%s, stdin=%s, shell=%s, universal_newlines=%s)", redacted_command, cwd, @@ -1073,6 +1439,7 @@ def execute( stdout=stdout_sink, shell=shell, universal_newlines=universal_newlines, + encoding=defenc if universal_newlines else None, **subprocess_kwargs, ) except cmd_not_found_exception as err: @@ -1085,37 +1452,64 @@ def execute( if as_process: return self.AutoInterrupt(proc, command) - def kill_process(pid: int) -> None: - """Callback to kill a process.""" - if os.name == "nt": - raise AssertionError("Bug: This callback would be ineffective and unsafe on Windows, stopping.") - p = Popen(["ps", "--ppid", str(pid)], stdout=PIPE) - child_pids = [] - if p.stdout is not None: - for line in p.stdout: - if len(line.split()) > 0: - local_pid = (line.split())[0] - if local_pid.isdigit(): - child_pids.append(int(local_pid)) - try: - os.kill(pid, signal.SIGKILL) - for child_pid in child_pids: - try: - os.kill(child_pid, signal.SIGKILL) - except OSError: - pass - kill_check.set() # Tell the main routine that the process was killed. - except OSError: - # It is possible that the process gets completed in the duration after - # timeout happens and before we try to kill the process. - pass - return + watchdog: Optional[threading.Timer] = None + kill_check: Optional[threading.Event] = None + timeout_error: Optional[Callable[[], Union[str, bytes]]] = None + + if sys.platform != "win32" and kill_after_timeout is not None: + # Help mypy figure out this is not None even when used inside communicate(). + timeout = kill_after_timeout + + def kill_process(pid: int) -> None: + """Callback to kill a process. + + This callback implementation would be ineffective and unsafe on Windows. + """ + p = Popen(["ps", "--ppid", str(pid)], stdout=PIPE) + child_pids = [] + if p.stdout is not None: + for line in p.stdout: + if len(line.split()) > 0: + local_pid = (line.split())[0] + if local_pid.isdigit(): + child_pids.append(int(local_pid)) + try: + os.kill(pid, signal.SIGKILL) + for child_pid in child_pids: + try: + os.kill(child_pid, signal.SIGKILL) + except OSError: + pass + # Tell the main routine that the process was killed. + assert kill_check is not None + kill_check.set() + except OSError: + # It is possible that the process gets completed in the duration + # after timeout happens and before we try to kill the process. + pass + return + + def make_timeout_error() -> Union[str, bytes]: + err = f'Timeout: the command "{" ".join(redacted_command)}" did not complete in {timeout:g} secs.' + return err if universal_newlines else err.encode(defenc) - # END kill_process + def communicate() -> Tuple[AnyStr, AnyStr]: + assert watchdog is not None + assert kill_check is not None + watchdog.start() + out, err = proc.communicate() + watchdog.cancel() + if kill_check.is_set(): + err = make_timeout_error() + return out, err + + # END helpers - if kill_after_timeout is not None: kill_check = threading.Event() - watchdog = threading.Timer(kill_after_timeout, kill_process, args=(proc.pid,)) + watchdog = threading.Timer(timeout, kill_process, args=(proc.pid,)) + timeout_error = make_timeout_error + else: + communicate = proc.communicate # Wait for the process to return. status = 0 @@ -1124,38 +1518,40 @@ def kill_process(pid: int) -> None: newline = "\n" if universal_newlines else b"\n" try: if output_stream is None: - if kill_after_timeout is not None: - watchdog.start() - stdout_value, stderr_value = proc.communicate() - if kill_after_timeout is not None: - watchdog.cancel() - if kill_check.is_set(): - stderr_value = 'Timeout: the command "%s" did not complete in %d ' "secs." % ( - " ".join(redacted_command), - kill_after_timeout, - ) - if not universal_newlines: - stderr_value = stderr_value.encode(defenc) + stdout_value, stderr_value = communicate() # Strip trailing "\n". - if stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore + if stdout_value is not None and stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore[arg-type] stdout_value = stdout_value[:-1] - if stderr_value.endswith(newline): # type: ignore + if stderr_value is not None and stderr_value.endswith(newline): # type: ignore[arg-type] stderr_value = stderr_value[:-1] status = proc.returncode else: max_chunk_size = max_chunk_size if max_chunk_size and max_chunk_size > 0 else io.DEFAULT_BUFFER_SIZE - stream_copy(proc.stdout, output_stream, max_chunk_size) - stdout_value = proc.stdout.read() - stderr_value = proc.stderr.read() + if watchdog is not None: + watchdog.start() + try: + if proc.stdout is not None: + stream_copy(proc.stdout, output_stream, max_chunk_size) + stdout_value = proc.stdout.read() + if proc.stderr is not None: + stderr_value = proc.stderr.read() + status = proc.wait() + finally: + if watchdog is not None: + watchdog.cancel() # Strip trailing "\n". - if stderr_value.endswith(newline): # type: ignore + if stderr_value is not None and stderr_value.endswith(newline): # type: ignore[arg-type] stderr_value = stderr_value[:-1] - status = proc.wait() + if kill_check is not None and kill_check.is_set(): + assert timeout_error is not None + stderr_value = timeout_error() # END stdout handling finally: - proc.stdout.close() - proc.stderr.close() + if proc.stdout is not None: + proc.stdout.close() + if proc.stderr is not None: + proc.stderr.close() if self.GIT_PYTHON_TRACE == "full": cmdstr = " ".join(redacted_command) @@ -1166,7 +1562,7 @@ def as_text(stdout_value: Union[bytes, str]) -> str: # END as_text if stderr_value: - log.info( + _logger.info( "%s -> %d; stdout: '%s'; stderr: '%s'", cmdstr, status, @@ -1174,9 +1570,9 @@ def as_text(stdout_value: Union[bytes, str]) -> str: safe_decode(stderr_value), ) elif stdout_value: - log.info("%s -> %d; stdout: '%s'", cmdstr, status, as_text(stdout_value)) + _logger.info("%s -> %d; stdout: '%s'", cmdstr, status, as_text(stdout_value)) else: - log.info("%s -> %d", cmdstr, status) + _logger.info("%s -> %d", cmdstr, status) # END handle debug printing if with_exceptions and status != 0: @@ -1199,14 +1595,16 @@ def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: values in a format that can be passed back into this function to revert the changes. - ``Examples``:: + Examples:: old_env = self.update_environment(PWD='/tmp') self.update_environment(**old_env) - :param kwargs: Environment variables to use for git processes + :param kwargs: + Environment variables to use for git processes. - :return: Dict that maps environment variables to their old values + :return: + Dict that maps environment variables to their old values """ old_env = {} for key, value in kwargs.items(): @@ -1225,12 +1623,13 @@ def custom_environment(self, **kwargs: Any) -> Iterator[None]: """A context manager around the above :meth:`update_environment` method to restore the environment back to its previous state after operation. - ``Examples``:: + Examples:: with self.custom_environment(GIT_SSH='/bin/ssh_wrapper'): repo.remotes.origin.fetch() - :param kwargs: See :meth:`update_environment` + :param kwargs: + See :meth:`update_environment`. """ old_env = self.update_environment(**kwargs) try: @@ -1255,7 +1654,7 @@ def transform_kwarg(self, name: str, value: Any, split_single_char_options: bool return [] def transform_kwargs(self, split_single_char_options: bool = True, **kwargs: Any) -> List[str]: - """Transform Python style kwargs into git command line options.""" + """Transform Python-style kwargs into git command line options.""" args = [] for k, v in kwargs.items(): if isinstance(v, (list, tuple)): @@ -1266,7 +1665,7 @@ def transform_kwargs(self, split_single_char_options: bool = True, **kwargs: Any return args @classmethod - def _unpack_args(cls, arg_list: Sequence[str]) -> List[str]: + def _unpack_args(cls, arg_list: Sequence[Any]) -> List[str]: outlist = [] if isinstance(arg_list, (list, tuple)): for arg in arg_list: @@ -1281,18 +1680,20 @@ def __call__(self, **kwargs: Any) -> "Git": :param kwargs: A dict of keyword arguments. - These arguments are passed as in :meth:`_call_process`, but will be - passed to the git command rather than the subcommand. + These arguments are passed as in :meth:`_call_process`, but will be passed + to the git command rather than the subcommand. + + Examples:: - ``Examples``:: git(work_tree='/tmp').difftool() """ self._git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) return self @overload - def _call_process(self, method: str, *args: None, **kwargs: None) -> str: - ... # If no args were given, execute the call with all defaults. + def _call_process( + self, method: str, *args: None, **kwargs: None + ) -> str: ... # If no args were given, execute the call with all defaults. @overload def _call_process( @@ -1302,35 +1703,36 @@ def _call_process( as_process: Literal[True], *args: Any, **kwargs: Any, - ) -> "Git.AutoInterrupt": - ... + ) -> "Git.AutoInterrupt": ... @overload def _call_process( self, method: str, *args: Any, **kwargs: Any - ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: - ... + ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: ... def _call_process( self, method: str, *args: Any, **kwargs: Any ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: - """Run the given git command with the specified arguments and return - the result as a string. + """Run the given git command with the specified arguments and return the result + as a string. :param method: - The command. Contained ``_`` characters will be converted to dashes, - such as in ``ls_files`` to call ``ls-files``. + The command. Contained ``_`` characters will be converted to hyphens, such + as in ``ls_files`` to call ``ls-files``. :param args: - The list of arguments. If None is included, it will be pruned. - This allows your commands to call git more conveniently as None - is realized as non-existent. + The list of arguments. If ``None`` is included, it will be pruned. + This allows your commands to call git more conveniently, as ``None`` is + realized as non-existent. :param kwargs: Contains key-values for the following: - - The :meth:`execute()` kwds, as listed in :var:`execute_kwargs`. + + - The :meth:`execute()` kwds, as listed in ``execute_kwargs``. - "Command options" to be converted by :meth:`transform_kwargs`. - - The `'insert_kwargs_after'` key which its value must match one of ``*args``. + - The ``insert_kwargs_after`` key which its value must match one of + ``*args``. + It also contains any command options, to be appended after the matched arg. Examples:: @@ -1339,11 +1741,12 @@ def _call_process( turns into:: - git rev-list max-count 10 --header master + git rev-list --max-count=10 --header master - :return: Same as :meth:`execute`. - If no args are given, used :meth:`execute`'s default (especially - ``as_process = False``, ``stdout_as_string = True``) and return str. + :return: + Same as :meth:`execute`. If no args are given, used :meth:`execute`'s + default (especially ``as_process = False``, ``stdout_as_string = True``) and + return :class:`str`. """ # Handle optional arguments prior to calling transform_kwargs. # Otherwise these'll end up in args, which is bad. @@ -1388,12 +1791,15 @@ def _call_process( def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: """ :param header_line: - type_string size_as_int + A line of the form:: + + type_string size_as_int - :return: (hex_sha, type_string, size_as_int) + :return: + (hex_sha, type_string, size_as_int) - :raise ValueError: If the header contains indication for an error due to - incorrect input sha + :raise ValueError: + If the header contains indication for an error due to incorrect input sha. """ tokens = header_line.split() if len(tokens) != 3: @@ -1449,22 +1855,27 @@ def __get_object_header(self, cmd: "Git.AutoInterrupt", ref: AnyStr) -> Tuple[st raise ValueError("cmd stdin was empty") def get_object_header(self, ref: str) -> Tuple[str, str, int]: - """Use this method to quickly examine the type and size of the object behind - the given ref. + """Use this method to quickly examine the type and size of the object behind the + given ref. - :note: The method will only suffer from the costs of command invocation - once and reuses the command in subsequent calls. + :note: + The method will only suffer from the costs of command invocation once and + reuses the command in subsequent calls. - :return: (hexsha, type_string, size_as_int) + :return: + (hexsha, type_string, size_as_int) """ cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) return self.__get_object_header(cmd, ref) def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: - """As get_object_header, but returns object data as well. + """Similar to :meth:`get_object_header`, but returns object data as well. + + :return: + (hexsha, type_string, size_as_int, data_string) - :return: (hexsha, type_string, size_as_int, data_string) - :note: Not threadsafe. + :note: + Not threadsafe. """ hexsha, typename, size, stream = self.stream_object_data(ref) data = stream.read(size) @@ -1472,10 +1883,14 @@ def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: return (hexsha, typename, size, data) def stream_object_data(self, ref: str) -> Tuple[str, str, int, "Git.CatFileContentStream"]: - """As get_object_header, but returns the data as a stream. + """Similar to :meth:`get_object_data`, but returns the data as a stream. - :return: (hexsha, type_string, size_as_int, stream) - :note: This method is not threadsafe, you need one independent Command instance per thread to be safe! + :return: + (hexsha, type_string, size_as_int, stream) + + :note: + This method is not threadsafe. You need one independent :class:`Git` + instance per thread to be safe! """ cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) @@ -1487,7 +1902,8 @@ def clear_cache(self) -> "Git": Currently persistent commands will be interrupted. - :return: self + :return: + self """ for cmd in (self.cat_file_all, self.cat_file_header): if cmd: diff --git a/git/compat.py b/git/compat.py index 920e44b7f..d7d9a55a9 100644 --- a/git/compat.py +++ b/git/compat.py @@ -3,24 +3,32 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Utilities to help provide compatibility with Python 3.""" +"""Utilities to help provide compatibility with Python 3. + +This module exists for historical reasons. Code outside GitPython may make use of public +members of this module, but is unlikely to benefit from doing so. GitPython continues to +use some of these utilities, in some cases for compatibility across different platforms. +""" import locale import os import sys +import warnings -from gitdb.utils.encoding import force_bytes, force_text # noqa: F401 # @UnusedImport +from gitdb.utils.encoding import force_bytes, force_text # noqa: F401 # typing -------------------------------------------------------------------- -from typing import ( # noqa: F401 - Any, +from typing import ( + Any, # noqa: F401 AnyStr, - Dict, - IO, + Dict, # noqa: F401 + IO, # noqa: F401 + List, Optional, - Tuple, - Type, + TYPE_CHECKING, + Tuple, # noqa: F401 + Type, # noqa: F401 Union, overload, ) @@ -28,37 +36,70 @@ # --------------------------------------------------------------------------- -is_win = os.name == "nt" +_deprecated_platform_aliases = { + "is_win": os.name == "nt", + "is_posix": os.name == "posix", + "is_darwin": sys.platform == "darwin", +} + + +def _getattr(name: str) -> Any: + try: + value = _deprecated_platform_aliases[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + + warnings.warn( + f"{__name__}.{name} and other is_ aliases are deprecated. " + "Write the desired os.name or sys.platform check explicitly instead.", + DeprecationWarning, + stacklevel=2, + ) + return value + + +if not TYPE_CHECKING: # Preserve static checking for undefined/misspelled attributes. + __getattr__ = _getattr + + +def __dir__() -> List[str]: + return [*globals(), *_deprecated_platform_aliases] + + +is_win: bool """Deprecated alias for ``os.name == "nt"`` to check for native Windows. This is deprecated because it is clearer to write out :attr:`os.name` or :attr:`sys.platform` checks explicitly, especially in cases where it matters which is used. -:note: ``is_win`` is ``False`` on Cygwin, but is often wrongly assumed ``True``. To - detect Cygwin, use ``sys.platform == "cygwin"``. +:note: + ``is_win`` is ``False`` on Cygwin, but is often wrongly assumed ``True``. To detect + Cygwin, use ``sys.platform == "cygwin"``. """ -is_posix = os.name == "posix" +is_posix: bool """Deprecated alias for ``os.name == "posix"`` to check for Unix-like ("POSIX") systems. This is deprecated because it clearer to write out :attr:`os.name` or :attr:`sys.platform` checks explicitly, especially in cases where it matters which is used. -:note: For POSIX systems, more detailed information is available in - :attr:`sys.platform`, while :attr:`os.name` is always ``"posix"`` on such systems, - including macOS (Darwin). +:note: + For POSIX systems, more detailed information is available in :attr:`sys.platform`, + while :attr:`os.name` is always ``"posix"`` on such systems, including macOS + (Darwin). """ -is_darwin = sys.platform == "darwin" +is_darwin: bool """Deprecated alias for ``sys.platform == "darwin"`` to check for macOS (Darwin). This is deprecated because it clearer to write out :attr:`os.name` or :attr:`sys.platform` checks explicitly. -:note: For macOS (Darwin), ``os.name == "posix"`` as in other Unix-like systems, while - ``sys.platform == "darwin"`. +:note: + For macOS (Darwin), ``os.name == "posix"`` as in other Unix-like systems, while + ``sys.platform == "darwin"``. """ defenc = sys.getfilesystemencoding() @@ -66,13 +107,11 @@ @overload -def safe_decode(s: None) -> None: - ... +def safe_decode(s: None) -> None: ... @overload -def safe_decode(s: AnyStr) -> str: - ... +def safe_decode(s: AnyStr) -> str: ... def safe_decode(s: Union[AnyStr, None]) -> Optional[str]: @@ -88,13 +127,11 @@ def safe_decode(s: Union[AnyStr, None]) -> Optional[str]: @overload -def safe_encode(s: None) -> None: - ... +def safe_encode(s: None) -> None: ... @overload -def safe_encode(s: AnyStr) -> bytes: - ... +def safe_encode(s: AnyStr) -> bytes: ... def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: @@ -110,13 +147,11 @@ def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: @overload -def win_encode(s: None) -> None: - ... +def win_encode(s: None) -> None: ... @overload -def win_encode(s: AnyStr) -> bytes: - ... +def win_encode(s: AnyStr) -> bytes: ... def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: diff --git a/git/config.py b/git/config.py index 2730ddaf3..e7f64f7b5 100644 --- a/git/config.py +++ b/git/config.py @@ -5,6 +5,8 @@ """Parser for reading and writing configuration files.""" +__all__ = ["GitConfigParser", "SectionConstraint"] + import abc import configparser as cp import fnmatch @@ -40,9 +42,10 @@ from git.types import Lit_config_levels, ConfigLevels_Tup, PathLike, assert_never, _T if TYPE_CHECKING: - from git.repo.base import Repo from io import BytesIO + from git.repo.base import Repo + T_ConfigParser = TypeVar("T_ConfigParser", bound="GitConfigParser") T_OMD_value = TypeVar("T_OMD_value", str, bytes, int, float, bool) @@ -58,44 +61,47 @@ # ------------------------------------------------------------- -__all__ = ("GitConfigParser", "SectionConstraint") - - -log = logging.getLogger("git.config") -log.addHandler(logging.NullHandler()) - +_logger = logging.getLogger(__name__) CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") """The configuration level of a configuration file.""" -CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") +CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch|hasconfig:remote\.\*\.url):(.+)\"") """Section pattern to detect conditional includes. See: https://git-scm.com/docs/git-config#_conditional_includes """ +UNSAFE_CONFIG_CHARS_RE = re.compile(r"[\r\n\x00]") +"""Characters that cannot be safely written in config names or values.""" + +VALID_CONFIG_OPTION_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +"""Pattern for option names that can be written without changing config syntax.""" + class MetaParserBuilder(abc.ABCMeta): # noqa: B024 - """Utility class wrapping base-class methods into decorators that assure read-only properties.""" + """Utility class wrapping base-class methods into decorators that assure read-only + properties.""" def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParserBuilder": """Equip all base-class methods with a needs_values decorator, and all non-const - methods with a set_dirty_and_flush_changes decorator in addition to that. + methods with a :func:`set_dirty_and_flush_changes` decorator in addition to + that. """ kmm = "_mutating_methods_" if kmm in clsdict: mutating_methods = clsdict[kmm] for base in bases: methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_")) - for name, method in methods: - if name in clsdict: + for method_name, method in methods: + if method_name in clsdict: continue method_with_values = needs_values(method) - if name in mutating_methods: + if method_name in mutating_methods: method_with_values = set_dirty_and_flush_changes(method_with_values) # END mutating methods handling - clsdict[name] = method_with_values + clsdict[method_name] = method_with_values # END for each name/method pair # END for each base # END if mutating methods configuration is set @@ -105,7 +111,8 @@ def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParse def needs_values(func: Callable[..., _T]) -> Callable[..., _T]: - """Return a method for ensuring we read values (on demand) before we try to access them.""" + """Return a method for ensuring we read values (on demand) before we try to access + them.""" @wraps(func) def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: @@ -119,7 +126,8 @@ def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _ def set_dirty_and_flush_changes(non_const_func: Callable[..., _T]) -> Callable[..., _T]: """Return a method that checks whether given non constant function may be called. - If so, the instance will be set dirty. Additionally, we flush the changes right to disk. + If so, the instance will be set dirty. Additionally, we flush the changes right to + disk. """ def flush_changes(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: @@ -139,7 +147,8 @@ class SectionConstraint(Generic[T_ConfigParser]): It supports all ConfigParser methods that operate on an option. - :note: If used as a context manager, will release the wrapped ConfigParser. + :note: + If used as a context manager, will release the wrapped ConfigParser. """ __slots__ = ("_config", "_section_name") @@ -174,8 +183,8 @@ def __getattr__(self, attr: str) -> Any: return super().__getattribute__(attr) def _call_config(self, method: str, *args: Any, **kwargs: Any) -> Any: - """Call the configuration at the given method which must take a section name - as first argument.""" + """Call the configuration at the given method which must take a section name as + first argument.""" return getattr(self._config, method)(self._section_name, *args, **kwargs) @property @@ -184,8 +193,8 @@ def config(self) -> T_ConfigParser: return self._config def release(self) -> None: - """Equivalent to GitConfigParser.release(), which is called on our underlying - parser instance.""" + """Equivalent to :meth:`GitConfigParser.release`, which is called on our + underlying parser instance.""" return self._config.release() def __enter__(self) -> "SectionConstraint[T_ConfigParser]": @@ -244,7 +253,7 @@ def items_all(self) -> List[Tuple[str, List[_T]]]: def get_config_path(config_level: Lit_config_levels) -> str: # We do not support an absolute path of the gitconfig on Windows. # Use the global config instead. - if os.name == "nt" and config_level == "system": + if sys.platform == "win32" and config_level == "system": config_level = "global" if config_level == "system": @@ -257,7 +266,8 @@ def get_config_path(config_level: Lit_config_levels) -> str: elif config_level == "repository": raise ValueError("No repo to get repository configuration from. Use Repo._get_config_path") else: - # Should not reach here. Will raise ValueError if does. Static typing will warn missing elifs + # Should not reach here. Will raise ValueError if does. Static typing will warn + # about missing elifs. assert_never( # type: ignore[unreachable] config_level, ValueError(f"Invalid configuration level: {config_level!r}"), @@ -267,14 +277,15 @@ def get_config_path(config_level: Lit_config_levels) -> str: class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): """Implements specifics required to read git style configuration files. - This variation behaves much like the git.config command such that the configuration - will be read on demand based on the filepath given during initialization. + This variation behaves much like the :manpage:`git-config(1)` command, such that the + configuration will be read on demand based on the filepath given during + initialization. The changes will automatically be written once the instance goes out of scope, but can be triggered manually as well. - The configuration file will be locked if you intend to change values preventing other - instances to write concurrently. + The configuration file will be locked if you intend to change values preventing + other instances to write concurrently. :note: The config is case-sensitive even when queried, hence section and option names @@ -288,7 +299,7 @@ class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): t_lock = LockFile """The lock type determines the type of lock to use in new configuration readers. - They must be compatible to the LockFile interface. + They must be compatible to the :class:`~git.util.LockFile` interface. A suitable alternative would be the :class:`~git.util.BlockingLockFile`. """ @@ -304,7 +315,8 @@ class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): del optvalueonly_source _mutating_methods_ = ("add_section", "remove_section", "remove_option", "set") - """List of RawConfigParser methods able to change the instance.""" + """Names of :class:`~configparser.RawConfigParser` methods able to change the + instance.""" def __init__( self, @@ -314,21 +326,22 @@ def __init__( config_level: Union[Lit_config_levels, None] = None, repo: Union["Repo", None] = None, ) -> None: - """Initialize a configuration reader to read the given file_or_files and to - possibly allow changes to it by setting read_only False. + """Initialize a configuration reader to read the given `file_or_files` and to + possibly allow changes to it by setting `read_only` False. :param file_or_files: A file path or file object, or a sequence of possibly more than one of them. :param read_only: - If True, the ConfigParser may only read the data, but not change it. - If False, only a single file path or file object may be given. We will write - back the changes when they happen, or when the ConfigParser is released. - This will not happen if other configuration files have been included. + If ``True``, the ConfigParser may only read the data, but not change it. + If ``False``, only a single file path or file object may be given. We will + write back the changes when they happen, or when the ConfigParser is + released. This will not happen if other configuration files have been + included. :param merge_includes: - If True, we will read files mentioned in ``[include]`` sections and merge - their contents into ours. This makes it impossible to write back an + If ``True``, we will read files mentioned in ``[include]`` sections and + merge their contents into ours. This makes it impossible to write back an individual configuration file. Thus, if you want to modify a single configuration file, turn this off to leave the original dataset unaltered when reading it. @@ -338,11 +351,12 @@ def __init__( configuration files. """ cp.RawConfigParser.__init__(self, dict_type=_OMD) - self._dict: Callable[..., _OMD] # type: ignore # mypy/typeshed bug? + self._dict: Callable[..., _OMD] self._defaults: _OMD - self._sections: _OMD # type: ignore # mypy/typeshed bug? + self._sections: _OMD - # Used in Python 3. Needs to stay in sync with sections for underlying implementation to work. + # Used in Python 3. Needs to stay in sync with sections for underlying + # implementation to work. if not hasattr(self, "_proxies"): self._proxies = self._dict() @@ -388,7 +402,7 @@ def _acquire_lock(self) -> None: # END read-only check def __del__(self) -> None: - """Write pending changes if required and release locks""" + """Write pending changes if required and release locks.""" # NOTE: Only consistent in Python 2. self.release() @@ -400,10 +414,12 @@ def __exit__(self, *args: Any) -> None: self.release() def release(self) -> None: - """Flush changes and release the configuration write lock. This instance must not be used anymore afterwards. + """Flush changes and release the configuration write lock. This instance must + not be used anymore afterwards. - In Python 3, it's required to explicitly release locks and flush changes, as __del__ is not called - deterministically anymore.""" + In Python 3, it's required to explicitly release locks and flush changes, as + ``__del__`` is not called deterministically anymore. + """ # Checking for the lock here makes sure we do not raise during write() # in case an invalid parser was created who could not get a lock. if self.read_only or (self._lock and not self._lock._has_lock()): @@ -412,7 +428,7 @@ def release(self) -> None: try: self.write() except IOError: - log.error("Exception during destruction of GitConfigParser", exc_info=True) + _logger.error("Exception during destruction of GitConfigParser", exc_info=True) except ReferenceError: # This happens in Python 3... and usually means that some state cannot be # written as the sections dict cannot be iterated. This usually happens when @@ -427,8 +443,9 @@ def optionxform(self, optionstr: str) -> str: return optionstr def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None: - """Originally a direct copy of the Python 2.4 version of RawConfigParser._read, - to ensure it uses ordered dicts. + """Originally a direct copy of the Python 2.4 version of + :meth:`RawConfigParser._read `, to ensure it + uses ordered dicts. The ordering bug was fixed in Python 2.4, and dict itself keeps ordering since Python 3.7. This has some other changes, especially that it ignores initial @@ -441,11 +458,12 @@ def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None: e = None # None, or an exception. def string_decode(v: str) -> str: - if v[-1] == "\\": + if v and v.endswith("\\"): v = v[:-1] # END cut trailing escapes to prevent decode error - return v.encode(defenc).decode("unicode_escape") + escapes = {"b": "\b", "n": "\n", "r": "\r", "t": "\t", '"': '"', "\\": "\\"} + return re.sub(r"\\(.)", lambda match: escapes.get(match.group(1), match.group(0)), v) # END string_decode @@ -485,19 +503,28 @@ def string_decode(v: str) -> str: if mo: # We might just have handled the last line, which could contain a quotation we want to remove. optname, vi, optval = mo.group("option", "vi", "value") + optname = self.optionxform(optname.rstrip()) + if vi in ("=", ":") and ";" in optval and not optval.strip().startswith('"'): pos = optval.find(";") if pos != -1 and optval[pos - 1].isspace(): optval = optval[:pos] optval = optval.strip() - if optval == '""': - optval = "" - # END handle empty string - optname = self.optionxform(optname.rstrip()) - if len(optval) > 1 and optval[0] == '"' and optval[-1] != '"': + + if len(optval) < 2 or optval[0] != '"': + # Does not open quoting. + pass + elif optval[-1] != '"': + # Opens quoting and does not close: appears to start multi-line quoting. is_multi_line = True optval = string_decode(optval[1:]) - # END handle multi-line + elif re.search(r'(?:^|[^\\])(?:\\\\)*"', optval[1:-1]): + # Preserve malformed values containing unescaped quotes. + pass + else: + # Opens and closes quoting. + optval = string_decode(optval[1:-1]) + # Preserves multiple values for duplicate optnames. cursect.add(optname, optval) else: @@ -528,13 +555,24 @@ def _has_includes(self) -> Union[bool, int]: def _included_paths(self) -> List[Tuple[str, str]]: """List all paths that must be included to configuration. - :return: The list of paths, where each path is a tuple of ``(option, value)``. + :return: + The list of paths, where each path is a tuple of (option, value). """ + + def _all_items(section: str) -> List[Tuple[str, str]]: + """Return all (key, value) pairs for a section, including duplicate keys.""" + return [ + (key, value) + for key, values in self._sections[section].items_all() + if key != "__name__" + for value in values + ] + paths = [] for section in self.sections(): if section == "include": - paths += self.items(section) + paths += _all_items(section) match = CONDITIONAL_INCLUDE_REGEXP.search(section) if match is None or self._repo is None: @@ -545,8 +583,12 @@ def _included_paths(self) -> List[Tuple[str, str]]: if keyword in ["gitdir", "gitdir/i"]: value = osp.expanduser(value) + git_dir = os.fspath(self._repo.git_dir) if self._repo.git_dir else None + if sys.platform == "win32": + git_dir = git_dir.replace("\\", "/") if git_dir else None - if not any(value.startswith(s) for s in ["./", "/"]): + drive, _tail = osp.splitdrive(value) + if not drive and not any(value.startswith(s) for s in ["./", "/"]): value = "**/" + value if value.endswith("/"): value += "**" @@ -555,12 +597,11 @@ def _included_paths(self) -> List[Tuple[str, str]]: if keyword.endswith("/i"): value = re.sub( r"[a-zA-Z]", - lambda m: "[{}{}]".format(m.group().lower(), m.group().upper()), + lambda m: f"[{m.group().lower()!r}{m.group().upper()!r}]", value, ) - if self._repo.git_dir: - if fnmatch.fnmatchcase(str(self._repo.git_dir), value): - paths += self.items(section) + if git_dir and fnmatch.fnmatchcase(git_dir, value): + paths += _all_items(section) elif keyword == "onbranch": try: @@ -570,8 +611,12 @@ def _included_paths(self) -> List[Tuple[str, str]]: continue if fnmatch.fnmatchcase(branch_name, value): - paths += self.items(section) - + paths += _all_items(section) + elif keyword == "hasconfig:remote.*.url": + for remote in self._repo.remotes: + if fnmatch.fnmatchcase(remote.url, value): + paths += _all_items(section) + break return paths def read(self) -> None: # type: ignore[override] @@ -580,8 +625,8 @@ def read(self) -> None: # type: ignore[override] This will ignore files that cannot be read, possibly leaving an empty configuration. - :return: Nothing - :raise IOError: If a file cannot be handled + :raise IOError: + If a file cannot be handled. """ if self._is_initialized: return @@ -594,10 +639,12 @@ def read(self) -> None: # type: ignore[override] elif not isinstance(self._file_or_files, (tuple, list, Sequence)): # Could merge with above isinstance once runtime type known. files_to_read = [self._file_or_files] - else: # for lists or tuples + else: # For lists or tuples. files_to_read = list(self._file_or_files) # END ensure we have a copy of the paths to handle + files_to_read = [osp.abspath(path) if isinstance(path, (str, os.PathLike)) else path for path in files_to_read] + seen = set(files_to_read) num_read_include_files = 0 while files_to_read: @@ -606,11 +653,10 @@ def read(self) -> None: # type: ignore[override] if hasattr(file_path, "seek"): # Must be a file-object. - file_path = cast(IO[bytes], file_path) # TODO: Replace with assert to narrow type, once sure. + # TODO: Replace cast with assert to narrow type, once sure. + file_path = cast(IO[bytes], file_path) self._read(file_path, file_path.name) else: - # Assume a path if it is not a file-object. - file_path = cast(PathLike, file_path) try: with open(file_path, "rb") as fp: file_ok = True @@ -618,8 +664,8 @@ def read(self) -> None: # type: ignore[override] except IOError: continue - # Read includes and append those that we didn't handle yet. - # We expect all paths to be normalized and absolute (and will ensure that is the case). + # Read includes and append those that we didn't handle yet. We expect all + # paths to be normalized and absolute (and will ensure that is the case). if self._has_includes(): for _, include_path in self._included_paths(): if include_path.startswith("~"): @@ -662,7 +708,11 @@ def write_section(name: str, section_dict: _OMD) -> None: continue for v in values: - fp.write(("\t%s = %s\n" % (key, self._value_to_string(v).replace("\n", "\n\t"))).encode(defenc)) + value = self._value_to_string(v) + if any(char in value for char in '\n\t\b\\"#;') or value[:1].isspace() or value[-1:].isspace(): + value = value.replace("\\", "\\\\").replace('"', '\\"') + value = '"%s\\\n"' % value.replace("\n", "\\n").replace("\t", "\\t").replace("\b", "\\b") + fp.write(("\t%s = %s\n" % (key, value)).encode(defenc)) # END if key is not __name__ # END section writing @@ -698,8 +748,10 @@ def items_all(self, section_name: str) -> List[Tuple[str, List[str]]]: def write(self) -> None: """Write changes to our file, if there are changes at all. - :raise IOError: If this is a read-only writer instance or if we could not obtain - a file lock""" + :raise IOError: + If this is a read-only writer instance or if we could not obtain a file + lock. + """ self._assure_writable("write") if not self._dirty: return @@ -712,13 +764,27 @@ def write(self) -> None: # END assert multiple files if self._has_includes(): - log.debug( + _logger.debug( "Skipping write-back of configuration file as include files were merged in." + "Set merge_includes=False to prevent this." ) return # END stop if we have include files + sections: List[_OMD] = [self._defaults] + section: _OMD + stored_section: _OMD + values: List[Any] + raw_value: Any + for _, stored_section in self._sections.items(): + sections.append(stored_section) + for section in sections: + for key, values in section.items_all(): + if key != "__name__": + for raw_value in values: + if "\r" in self._value_to_string(raw_value) or "\x00" in self._value_to_string(raw_value): + raise ValueError("Git config values must not contain CR or NUL") + fp = self._file_or_files # We have a physical file on disk, so get a lock. @@ -742,13 +808,14 @@ def _assure_writable(self, method_name: str) -> None: if self.read_only: raise IOError("Cannot execute non-constant method %s.%s" % (self, method_name)) - def add_section(self, section: str) -> None: - """Assures added options will stay in order""" + def add_section(self, section: "cp._SectionName") -> None: + """Assures added options will stay in order.""" + self._assure_config_name_safe(section, "section") return super().add_section(section) @property def read_only(self) -> bool: - """:return: True if this instance may change the configuration file""" + """:return: ``True`` if this instance may change the configuration file""" return self._read_only # FIXME: Figure out if default or return type can really include bool. @@ -760,16 +827,18 @@ def get_value( ) -> Union[int, float, str, bool]: """Get an option's value. - If multiple values are specified for this option in the section, the - last one specified is returned. + If multiple values are specified for this option in the section, the last one + specified is returned. :param default: - If not None, the given default value will be returned in case - the option did not exist + If not ``None``, the given default value will be returned in case the option + did not exist. - :return: a properly typed value, either int, float or string + :return: + A properly typed value, either int, float or string - :raise TypeError: in case the value could not be understood + :raise TypeError: + In case the value could not be understood. Otherwise the exceptions known to the ConfigParser will be raised. """ try: @@ -793,12 +862,14 @@ def get_values( returned. :param default: - If not None, a list containing the given default value will be - returned in case the option did not exist + If not ``None``, a list containing the given default value will be returned + in case the option did not exist. - :return: a list of properly typed values, either int, float or string + :return: + A list of properly typed values, either int, float or string - :raise TypeError: in case the value could not be understood + :raise TypeError: + In case the value could not be understood. Otherwise the exceptions known to the ConfigParser will be raised. """ try: @@ -844,23 +915,74 @@ def _value_to_string(self, value: Union[str, bytes, int, float, bool]) -> str: return str(value) return force_text(value) + def _value_to_string_safe(self, value: Union[str, bytes, int, float, bool]) -> str: + value_str = self._value_to_string(value) + if UNSAFE_CONFIG_CHARS_RE.search(value_str): + raise ValueError("Git config values must not contain CR, LF, or NUL") + return value_str + + def _assure_config_name_safe(self, name: "cp._SectionName", label: str) -> None: + if isinstance(name, str) and UNSAFE_CONFIG_CHARS_RE.search(name): + raise ValueError("Git config %s names must not contain CR, LF, or NUL" % label) + if label == "option" and isinstance(name, str) and not VALID_CONFIG_OPTION_NAME_RE.fullmatch(name): + raise ValueError("Git config option names may contain only letters, digits, '-', '_', or '.'") + if label == "section" and isinstance(name, str): + in_quotes = False + escaped = False + for index, char in enumerate(name): + if escaped: + escaped = False + elif in_quotes and char == "\\": + escaped = True + elif char == '"': + if not in_quotes and (index == 0 or name[index - 1] not in " \t"): + raise ValueError("Git config quoted subsection names must begin after whitespace") + in_quotes = not in_quotes + elif char == "]" and not in_quotes: + raise ValueError("Git config section names must not contain an unquoted closing bracket") + if in_quotes: + raise ValueError("Git config section names must not contain an unterminated quote") + + @needs_values + @set_dirty_and_flush_changes + def set( + self, + section: str, + option: str, + value: Union[str, bytes, int, float, bool, None] = None, + ) -> None: + self._assure_config_name_safe(section, "section") + self._assure_config_name_safe(option, "option") + if value is not None: + value = self._value_to_string_safe(value) + return super().set(section, option, value) + @needs_values @set_dirty_and_flush_changes def set_value(self, section: str, option: str, value: Union[str, bytes, int, float, bool]) -> "GitConfigParser": """Set the given option in section to the given value. This will create the section if required, and will not throw as opposed to the - default ConfigParser 'set' method. + default ConfigParser ``set`` method. - :param section: Name of the section in which the option resides or should reside - :param option: Name of the options whose value to set - :param value: Value to set the option to. It must be a string or convertible to - a string. - :return: This instance + :param section: + Name of the section in which the option resides or should reside. + + :param option: + Name of the options whose value to set. + + :param value: + Value to set the option to. It must be a string or convertible to a string. + + :return: + This instance """ + self._assure_config_name_safe(section, "section") + self._assure_config_name_safe(option, "option") + value_str = self._value_to_string_safe(value) if not self.has_section(section): self.add_section(section) - self.set(section, option, self._value_to_string(value)) + super().set(section, option, value_str) return self @needs_values @@ -868,30 +990,46 @@ def set_value(self, section: str, option: str, value: Union[str, bytes, int, flo def add_value(self, section: str, option: str, value: Union[str, bytes, int, float, bool]) -> "GitConfigParser": """Add a value for the given option in section. - This will create the section if required, and will not throw as opposed to the default - ConfigParser 'set' method. The value becomes the new value of the option as returned - by 'get_value', and appends to the list of values returned by 'get_values`'. + This will create the section if required, and will not throw as opposed to the + default ConfigParser ``set`` method. The value becomes the new value of the + option as returned by :meth:`get_value`, and appends to the list of values + returned by :meth:`get_values`. + + :param section: + Name of the section in which the option resides or should reside. - :param section: Name of the section in which the option resides or should reside - :param option: Name of the option - :param value: Value to add to option. It must be a string or convertible - to a string - :return: This instance + :param option: + Name of the option. + + :param value: + Value to add to option. It must be a string or convertible to a string. + + :return: + This instance """ + self._assure_config_name_safe(section, "section") + self._assure_config_name_safe(option, "option") + value_str = self._value_to_string_safe(value) if not self.has_section(section): self.add_section(section) - self._sections[section].add(option, self._value_to_string(value)) + self._sections[section].add(option, value_str) return self def rename_section(self, section: str, new_name: str) -> "GitConfigParser": - """Rename the given section to new_name. + """Rename the given section to `new_name`. + + :raise ValueError: + If: + + * `section` doesn't exist. + * A section with `new_name` does already exist. - :raise ValueError: If ``section`` doesn't exist - :raise ValueError: If a section with ``new_name`` does already exist - :return: This instance + :return: + This instance """ if not self.has_section(section): raise ValueError("Source section '%s' doesn't exist" % section) + self._assure_config_name_safe(new_name, "section") if self.has_section(new_name): raise ValueError("Destination section '%s' already exists" % new_name) @@ -901,6 +1039,7 @@ def rename_section(self, section: str, new_name: str) -> "GitConfigParser": new_section.setall(k, vs) # END for each value to copy - # This call writes back the changes, which is why we don't have the respective decorator. + # This call writes back the changes, which is why we don't have the respective + # decorator. self.remove_section(section) return self diff --git a/git/db.py b/git/db.py index 03b631084..bd68a5157 100644 --- a/git/db.py +++ b/git/db.py @@ -3,36 +3,36 @@ """Module with our own gitdb implementation - it uses the git command.""" -from git.util import bin_to_hex, hex_to_bin -from gitdb.base import OInfo, OStream -from gitdb.db import GitDB -from gitdb.db import LooseObjectDB +__all__ = ["GitCmdObjectDB", "GitDB"] + +from subprocess import PIPE +from gitdb.base import IStream, OInfo, OStream +from gitdb.db import GitDB, LooseObjectDB from gitdb.exc import BadObject +from gitdb.fun import stream_copy + +from git.compat import force_text +from git.util import bin_to_hex, hex_to_bin from git.exc import GitCommandError # typing------------------------------------------------- from typing import TYPE_CHECKING + from git.types import PathLike if TYPE_CHECKING: from git.cmd import Git - # -------------------------------------------------------- -__all__ = ("GitCmdObjectDB", "GitDB") - class GitCmdObjectDB(LooseObjectDB): """A database representing the default git object store, which includes loose objects, pack files and an alternates file. It will create objects only in the loose object database. - - :note: For now, we use the git command to do all the lookup, just until we - have packs and the other implementations. """ def __init__(self, root_path: PathLike, git: "Git") -> None: @@ -41,25 +41,48 @@ def __init__(self, root_path: PathLike, git: "Git") -> None: self._git = git def info(self, binsha: bytes) -> OInfo: + """Get a git object header (using git itself).""" hexsha, typename, size = self._git.get_object_header(bin_to_hex(binsha)) return OInfo(hex_to_bin(hexsha), typename, size) def stream(self, binsha: bytes) -> OStream: - """For now, all lookup is done by git itself""" + """Get git object data as a stream supporting ``read()`` (using git itself).""" hexsha, typename, size, stream = self._git.stream_object_data(bin_to_hex(binsha)) return OStream(hex_to_bin(hexsha), typename, size, stream) + def store(self, istream: IStream) -> IStream: + """Store an object using git itself.""" + if istream.binsha is not None or self.ostream() is not None: + return super().store(istream) + + proc = self._git.hash_object( + "-t", force_text(istream.type), "-w", "--stdin", "--literally", as_process=True, istream=PIPE + ) + assert proc.stdin is not None + try: + stream_copy(istream.read, proc.stdin.write, istream.size, self.stream_chunk_size) + finally: + proc.stdin.close() + assert proc.stdout is not None + hexsha = proc.stdout.read().strip() + proc.wait() + istream.binsha = hex_to_bin(hexsha) + return istream + # { Interface def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes: """ - :return: Full binary 20 byte sha from the given partial hexsha + :return: + Full binary 20 byte sha from the given partial hexsha + + :raise gitdb.exc.AmbiguousObjectName: - :raise AmbiguousObjectName: - :raise BadObject: + :raise gitdb.exc.BadObject: - :note: Currently we only raise :class:`BadObject` as git does not communicate - AmbiguousObjects separately. + :note: + Currently we only raise :exc:`~gitdb.exc.BadObject` as git does not + communicate ambiguous objects separately. """ try: hexsha, _typename, _size = self._git.get_object_header(partial_hexsha) diff --git a/git/diff.py b/git/diff.py index aba1a1080..192c099d0 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,16 +3,18 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["DiffConstants", "NULL_TREE", "NULL_TREE_SHA", "INDEX", "Diffable", "DiffIndex", "Diff"] + +import enum import re +import warnings -from git.cmd import handle_process_output +from git.cmd import Git, handle_process_output from git.compat import defenc +from git.objects.blob import Blob +from git.objects.util import mode_str_to_int from git.util import finalize_process, hex_to_bin -from .objects.blob import Blob -from .objects.util import mode_str_to_int - - # typing ------------------------------------------------------------------ from typing import ( @@ -21,46 +23,107 @@ List, Match, Optional, + Sequence, Tuple, - Type, + TYPE_CHECKING, TypeVar, Union, - TYPE_CHECKING, cast, ) from git.types import PathLike, Literal if TYPE_CHECKING: - from .objects.tree import Tree - from .objects import Commit - from git.repo.base import Repo - from git.objects.base import IndexObject from subprocess import Popen - from git import Git + + from git.objects.base import IndexObject + from git.objects.commit import Commit + from git.objects.tree import Tree + from git.repo.base import Repo Lit_change_type = Literal["A", "D", "C", "M", "R", "T", "U"] +# ------------------------------------------------------------------------ -# def is_change_type(inp: str) -> TypeGuard[Lit_change_type]: -# # return True -# return inp in ['A', 'D', 'C', 'M', 'R', 'T', 'U'] -# ------------------------------------------------------------------------ +@enum.unique +class DiffConstants(enum.Enum): + """Special objects for :meth:`Diffable.diff`. + See the :meth:`Diffable.diff` method's ``other`` parameter, which accepts various + values including these. -__all__ = ("Diffable", "DiffIndex", "Diff", "NULL_TREE") + :note: + These constants are also available as attributes of the :mod:`git.diff` module, + the :class:`Diffable` class and its subclasses and instances, and the top-level + :mod:`git` module. + """ -NULL_TREE = object() -"""Special object to compare against the empty tree in diffs.""" + NULL_TREE = enum.auto() + """Stand-in indicating you want to compare against the empty tree in diffs. -_octal_byte_re = re.compile(rb"\\([0-9]{3})") + Also accessible as :const:`git.NULL_TREE`, :const:`git.diff.NULL_TREE`, and + :const:`Diffable.NULL_TREE`. + """ + INDEX = enum.auto() + """Stand-in indicating you want to diff against the index. -def _octal_repl(matchobj: Match) -> bytes: - value = matchobj.group(1) - value = int(value, 8) - value = bytes(bytearray((value,))) - return value + Also accessible as :const:`git.INDEX`, :const:`git.diff.INDEX`, and + :const:`Diffable.INDEX`, as well as :const:`Diffable.Index`. The latter has been + kept for backward compatibility and made an alias of this, so it may still be used. + """ + + +NULL_TREE: Literal[DiffConstants.NULL_TREE] = DiffConstants.NULL_TREE +"""Stand-in indicating you want to compare against the empty tree in diffs. + +See :meth:`Diffable.diff`, which accepts this as a value of its ``other`` parameter. + +This is an alias of :const:`DiffConstants.NULL_TREE`, which may also be accessed as +:const:`git.NULL_TREE` and :const:`Diffable.NULL_TREE`. +""" + +NULL_TREE_SHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" +"""SHA of Git's canonical empty tree object.""" + +INDEX: Literal[DiffConstants.INDEX] = DiffConstants.INDEX +"""Stand-in indicating you want to diff against the index. + +See :meth:`Diffable.diff`, which accepts this as a value of its ``other`` parameter. + +This is an alias of :const:`DiffConstants.INDEX`, which may also be accessed as +:const:`git.INDEX` and :const:`Diffable.INDEX`, as well as :const:`Diffable.Index`. +""" + + +def _unquote_path(path: bytes) -> bytes: + result = bytearray() + escapes = { + ord("a"): 7, + ord("b"): 8, + ord("f"): 12, + ord("n"): 10, + ord("r"): 13, + ord("t"): 9, + ord("v"): 11, + } + i = 0 + while i < len(path): + if path[i] != ord("\\") or i + 1 == len(path): + result.append(path[i]) + i += 1 + continue + if path[i + 1] in b"0123" and i + 3 < len(path) and all(c in b"01234567" for c in path[i + 2 : i + 4]): + result.append(int(path[i + 1 : i + 4], 8)) + i += 4 + continue + escaped = path[i + 1] + if escaped in escapes or escaped in b'\\"': + result.append(escapes.get(escaped, escaped)) + else: + result.extend(path[i : i + 2]) + i += 2 + return bytes(result) def decode_path(path: bytes, has_ab_prefix: bool = True) -> Optional[bytes]: @@ -68,9 +131,7 @@ def decode_path(path: bytes, has_ab_prefix: bool = True) -> Optional[bytes]: return None if path.startswith(b'"') and path.endswith(b'"'): - path = path[1:-1].replace(b"\\n", b"\n").replace(b"\\t", b"\t").replace(b'\\"', b'"').replace(b"\\\\", b"\\") - - path = _octal_byte_re.sub(_octal_repl, path) + path = _unquote_path(path[1:-1]) if has_ab_prefix: assert path.startswith(b"a/") or path.startswith(b"b/") @@ -84,18 +145,56 @@ class Diffable: compatible type. :note: - Subclasses require a repo member as it is the case for Object instances, for - practical reasons we do not derive from Object. + Subclasses require a :attr:`repo` member, as it is the case for + :class:`~git.objects.base.Object` instances. For practical reasons we do not + derive from :class:`~git.objects.base.Object`. """ __slots__ = () - class Index: - """Stand-in indicating you want to diff against the index.""" + repo: "Repo" + """Repository to operate on. Must be provided by subclass or sibling class.""" + + NULL_TREE = NULL_TREE + """Stand-in indicating you want to compare against the empty tree in diffs. + + See the :meth:`diff` method, which accepts this as a value of its ``other`` + parameter. + + This is the same as :const:`DiffConstants.NULL_TREE`, and may also be accessed as + :const:`git.NULL_TREE` and :const:`git.diff.NULL_TREE`. + """ + + INDEX = INDEX + """Stand-in indicating you want to diff against the index. + + See the :meth:`diff` method, which accepts this as a value of its ``other`` + parameter. + + This is the same as :const:`DiffConstants.INDEX`, and may also be accessed as + :const:`git.INDEX` and :const:`git.diff.INDEX`, as well as :class:`Diffable.INDEX`, + which is kept for backward compatibility (it is now defined an alias of this). + """ + + Index = INDEX + """Stand-in indicating you want to diff against the index + (same as :const:`~Diffable.INDEX`). + + This is an alias of :const:`~Diffable.INDEX`, for backward compatibility. See + :const:`~Diffable.INDEX` and :meth:`diff` for details. + + :note: + Although always meant for use as an opaque constant, this was formerly defined + as a class. Its usage is unchanged, but static type annotations that attempt + to permit only this object must be changed to avoid new mypy errors. This was + previously not possible to do, though ``Type[Diffable.Index]`` approximated it. + It is now possible to do precisely, using ``Literal[DiffConstants.INDEX]``. + """ def _process_diff_args( - self, args: List[Union[str, "Diffable", Type["Diffable.Index"], object]] - ) -> List[Union[str, "Diffable", Type["Diffable.Index"], object]]: + self, + args: List[Union[PathLike, "Diffable"]], + ) -> List[Union[PathLike, "Diffable"]]: """ :return: Possibly altered version of the given args list. @@ -106,46 +205,63 @@ def _process_diff_args( def diff( self, - other: Union[Type["Index"], "Tree", "Commit", None, str, object] = Index, + other: Union[DiffConstants, "Tree", "Commit", str, None] = INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, + allow_unsafe_options: bool = False, **kwargs: Any, - ) -> "DiffIndex": - """Create diffs between two items being trees, trees and index or an - index and the working tree. Detects renames automatically. + ) -> "DiffIndex[Diff]": + """Create diffs between two items being trees, trees and index or an index and + the working tree. Detects renames automatically. :param other: This the item to compare us with. - * If None, we will be compared to the working tree. - * If :class:`~git.index.base.Treeish`, it will be compared against the - respective tree. - * If :class:`~Diffable.Index`, it will be compared against the index. - * If :attr:`git.NULL_TREE`, it will compare against the empty tree. - * It defaults to :class:`~Diffable.Index` so that the method will not by - default fail on bare repositories. + * If ``None``, we will be compared to the working tree. + + * If a :class:`~git.types.Tree_ish` or string, it will be compared against + the respective tree. + + * If :const:`INDEX`, it will be compared against the index. + + * If :const:`NULL_TREE`, it will compare against the empty tree. + + This parameter defaults to :const:`INDEX` (rather than ``None``) so that the + method will not by default fail on bare repositories. :param paths: This a list of paths or a single path to limit the diff to. It will only include at least one of the given path or paths. :param create_patch: - If True, the returned :class:`Diff` contains a detailed patch that if + If ``True``, the returned :class:`Diff` contains a detailed patch that if applied makes the self to other. Patches are somewhat costly as blobs have to be read and diffed. + :param allow_unsafe_options: + If ``True``, allow options such as ``--output``, ``--no-index``, and ``-O`` + that can write to or read from arbitrary filesystem paths. + :param kwargs: - Additional arguments passed to git-diff, such as ``R=True`` to swap both - sides of the diff. + Additional arguments passed to :manpage:`git-diff(1)`, such as ``R=True`` to + swap both sides of the diff. - :return: git.DiffIndex + :return: + A :class:`DiffIndex` representing the computed diff. :note: - On a bare repository, 'other' needs to be provided as - :class:`~Diffable.Index`, or as :class:`~git.objects.tree.Tree` or + On a bare repository, `other` needs to be provided as :const:`INDEX`, or as + an instance of :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit`, or a git command error will occur. """ - args: List[Union[PathLike, Diffable, Type["Diffable.Index"], object]] = [] + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([other], kwargs), + unsafe_options=self.repo.unsafe_git_diff_options, + clusterable_short_options="46abceflmnpqrstuvwzBCDMNRW", + ) + + args: List[Union[PathLike, Diffable]] = [] args.append("--abbrev=40") # We need full shas. args.append("--full-index") # Get full index paths, not only filenames. @@ -155,6 +271,7 @@ def diff( if create_patch: args.append("-p") + args.append("--no-ext-diff") else: args.append("--raw") args.append("-z") @@ -166,11 +283,8 @@ def diff( if paths is not None and not isinstance(paths, (tuple, list)): paths = [paths] - if hasattr(self, "Has_Repo"): - self.repo: "Repo" = self.repo - diff_cmd = self.repo.git.diff - if other is self.Index: + if other is INDEX: args.insert(0, "--cached") elif other is NULL_TREE: args.insert(0, "-r") # Recursive diff-tree. @@ -183,14 +297,17 @@ def diff( args.insert(0, self) - # paths is list here, or None. + # paths is a list or tuple here, or None. if paths: args.append("--") args.extend(paths) # END paths handling kwargs["as_process"] = True - proc = diff_cmd(*self._process_diff_args(args), **kwargs) + args = self._process_diff_args(args) + if create_patch: + self.repo.git(c="diff.mnemonicPrefix=false") + proc = diff_cmd(*args, **kwargs) diff_method = Diff._index_from_patch_format if create_patch else Diff._index_from_raw_format index = diff_method(self.repo, proc) @@ -203,13 +320,13 @@ def diff( class DiffIndex(List[T_Diff]): - """An Index for diffs, allowing a list of Diffs to be queried by the diff + R"""An index for diffs, allowing a list of :class:`Diff`\s to be queried by the diff properties. The class improves the diff handling convenience. """ - change_type = ("A", "C", "D", "R", "M", "T") + change_type: Sequence[Literal["A", "C", "D", "R", "M", "T"]] = ("A", "C", "D", "R", "M", "T") # noqa: F821 """Change type invariant identifying possible ways a blob can have changed: * ``A`` = Added @@ -245,7 +362,7 @@ def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: yield diffidx elif change_type == "C" and diffidx.copied_file: yield diffidx - elif change_type == "R" and diffidx.renamed: + elif change_type == "R" and diffidx.renamed_file: yield diffidx elif change_type == "M" and diffidx.a_blob and diffidx.b_blob and diffidx.a_blob != diffidx.b_blob: yield diffidx @@ -255,34 +372,34 @@ def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: class Diff: """A Diff contains diff information between two Trees. - It contains two sides a and b of the diff, members are prefixed with - "a" and "b" respectively to indicate that. + It contains two sides a and b of the diff. Members are prefixed with "a" and "b" + respectively to indicate that. Diffs keep information about the changed blob objects, the file mode, renames, deletions and new files. - There are a few cases where None has to be expected as member variable value: + There are a few cases where ``None`` has to be expected as member variable value: - ``New File``:: + New File:: a_mode is None a_blob is None a_path is None - ``Deleted File``:: + Deleted File:: b_mode is None b_blob is None b_path is None - ``Working Tree Blobs`` + Working Tree Blobs: When comparing to working trees, the working tree blob will have a null hexsha - as a corresponding object does not yet exist. The mode will be null as well. - The path will be available, though. + as a corresponding object does not yet exist. The mode will be null as well. The + path will be available, though. - If it is listed in a diff, the working tree version of the file must - differ from the version in the index or tree, and hence has been modified. + If it is listed in a diff, the working tree version of the file must differ from + the version in the index or tree, and hence has been modified. """ # Precompiled regex. @@ -466,17 +583,25 @@ def rename_to(self) -> Optional[str]: @property def renamed(self) -> bool: - """ - :return: True if the blob of our diff has been renamed + """Deprecated, use :attr:`renamed_file` instead. + + :return: + ``True`` if the blob of our diff has been renamed - :note: This property is deprecated. + :note: + This property is deprecated. Please use the :attr:`renamed_file` property instead. """ + warnings.warn( + "Diff.renamed is deprecated, use Diff.renamed_file instead", + DeprecationWarning, + stacklevel=2, + ) return self.renamed_file @property def renamed_file(self) -> bool: - """:return: True if the blob of our diff has been renamed""" + """:return: ``True`` if the blob of our diff has been renamed""" return self.rename_from != self.rename_to @classmethod @@ -493,17 +618,31 @@ def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_m return None @classmethod - def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex: - """Create a new DiffIndex from the given process output which must be in patch format. + def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex["Diff"]: + """Create a new :class:`DiffIndex` from the given process output which must be + in patch format. + + :param repo: + The repository we are operating on. - :param repo: The repository we are operating on - :param proc: ``git diff`` process to read from (supports :class:`Git.AutoInterrupt` wrapper) - :return: git.DiffIndex + :param proc: + :manpage:`git-diff(1)` process to read from + (supports :class:`Git.AutoInterrupt ` wrapper). + + :return: + :class:`DiffIndex` """ # FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. text_list: List[bytes] = [] - handle_process_output(proc, text_list.append, None, finalize_process, decode_streams=False) + stderr_list: List[bytes] = [] + + def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> None: + finalize_process(proc, stderr=b"".join(stderr_list)) + + handle_process_output( + proc, text_list.append, stderr_list.append, finalize_process_with_stderr, decode_streams=False + ) # For now, we have to bake the stream. text = b"".join(text_list) @@ -539,14 +678,14 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn a_path = cls._pick_best_path(a_path, rename_from, a_path_fallback) b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback) - # Our only means to find the actual text is to see what has not been matched by our regex, - # and then retro-actively assign it to our index. + # Our only means to find the actual text is to see what has not been matched + # by our regex, and then retro-actively assign it to our index. if previous_header is not None: index[-1].diff = text[previous_header.end() : _header.start()] # END assign actual diff - # Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid. - # We just use the one mode we should have parsed. + # Make sure the mode is set if the path is set. Otherwise the resulting blob + # is invalid. We just use the one mode we should have parsed. a_mode = old_mode or deleted_file_mode or (a_path and (b_mode or new_mode or new_file_mode)) b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode) index.append( @@ -579,7 +718,7 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn return index @staticmethod - def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> None: + def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]) -> None: lines = lines_bytes.decode(defenc) # Discard everything before the first colon, and the colon itself. @@ -597,11 +736,10 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non # Change type can be R100 # R: status letter # 100: score (in case of copy and rename) - # assert is_change_type(_change_type[0]), f"Unexpected value for change_type received: {_change_type[0]}" change_type: Lit_change_type = cast(Lit_change_type, _change_type[0]) score_str = "".join(_change_type[1:]) score = int(score_str) if score_str.isdigit() else None - path = path.strip() + path = path.strip("\n") a_path = path.encode(defenc) b_path = path.encode(defenc) deleted_file = False @@ -610,7 +748,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non rename_from = None rename_to = None - # NOTE: We cannot conclude from the existence of a blob to change type + # NOTE: We cannot conclude from the existence of a blob to change type, # as diffs with the working do not have blobs yet. if change_type == "D": b_blob_id = None # Optional[str] @@ -653,22 +791,33 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non index.append(diff) @classmethod - def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex": - """Create a new DiffIndex from the given process output which must be in raw format. + def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex[Diff]": + """Create a new :class:`DiffIndex` from the given process output which must be + in raw format. + + :param repo: + The repository we are operating on. + + :param proc: + Process to read output from. - :param repo: The repository we are operating on - :param proc: Process to read output from - :return: git.DiffIndex + :return: + :class:`DiffIndex` """ # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore index: "DiffIndex" = DiffIndex() + stderr_list: List[bytes] = [] + + def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> None: + finalize_process(proc, stderr=b"".join(stderr_list)) + handle_process_output( proc, lambda byt: cls._handle_diff_line(byt, repo, index), - None, - finalize_process, + stderr_list.append, + finalize_process_with_stderr, decode_streams=False, ) diff --git a/git/exc.py b/git/exc.py index 85113bc44..583eee8c1 100644 --- a/git/exc.py +++ b/git/exc.py @@ -42,12 +42,14 @@ ParseError, UnsupportedOperation, ) + from git.compat import safe_decode from git.util import remove_password_if_present # typing ---------------------------------------------------- -from typing import List, Sequence, Tuple, Union, TYPE_CHECKING +from typing import List, Sequence, Tuple, TYPE_CHECKING, Union + from git.types import PathLike if TYPE_CHECKING: @@ -81,16 +83,20 @@ class UnsafeOptionError(GitError): class CommandError(GitError): - """Base class for exceptions thrown at every stage of `Popen()` execution. + """Base class for exceptions thrown at every stage of :class:`~subprocess.Popen` + execution. :param command: A non-empty list of argv comprising the command-line. """ - #: A unicode print-format with 2 `%s for `` and the rest, - #: e.g. - #: "'%s' failed%s" _msg = "Cmd('%s') failed%s" + """Format string with 2 ``%s`` for ```` and the rest. + + For example: ``"'%s' failed%s"`` + + Subclasses may override this attribute, provided it is still in this form. + """ def __init__( self, @@ -132,8 +138,8 @@ def __str__(self) -> str: class GitCommandNotFound(CommandError): - """Thrown if we cannot find the `git` executable in the PATH or at the path given by - the GIT_PYTHON_GIT_EXECUTABLE environment variable.""" + """Thrown if we cannot find the ``git`` executable in the :envvar:`PATH` or at the + path given by the :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable.""" def __init__(self, command: Union[List[str], Tuple[str], str], cause: Union[str, Exception]) -> None: super().__init__(command, cause) @@ -184,7 +190,7 @@ def __str__(self) -> str: class CacheError(GitError): - """Base for all errors related to the git index, which is called cache + """Base for all errors related to the git index, which is called "cache" internally.""" diff --git a/git/ext/gitdb b/git/ext/gitdb deleted file mode 160000 index 3d3e9572d..000000000 --- a/git/ext/gitdb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3d3e9572dc452fea53d328c101b3d1440bbefe40 diff --git a/git/index/__init__.py b/git/index/__init__.py index c65722cd8..ba48110fd 100644 --- a/git/index/__init__.py +++ b/git/index/__init__.py @@ -3,5 +3,14 @@ """Initialize the index package.""" -from .base import * # noqa: F401 F403 -from .typ import * # noqa: F401 F403 +__all__ = [ + "BaseIndexEntry", + "BlobFilter", + "CheckoutError", + "IndexEntry", + "IndexFile", + "StageType", +] + +from .base import CheckoutError, IndexFile +from .typ import BaseIndexEntry, BlobFilter, IndexEntry, StageType diff --git a/git/index/base.py b/git/index/base.py index 9d1db4e35..a3c915242 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -3,61 +3,55 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module containing IndexFile, an Index implementation facilitating all kinds of index -manipulations such as querying and merging.""" +"""Module containing :class:`IndexFile`, an Index implementation facilitating all kinds +of index manipulations such as querying and merging.""" + +__all__ = ["IndexFile", "CheckoutError", "StageType"] import contextlib import datetime import glob from io import BytesIO import os +import os.path as osp from stat import S_ISLNK import subprocess +import sys import tempfile -from git.compat import ( - force_bytes, - defenc, -) -from git.exc import GitCommandError, CheckoutError, GitError, InvalidGitRepositoryError -from git.objects import ( - Blob, - Submodule, - Tree, - Object, - Commit, -) +from gitdb.base import IStream +from gitdb.db import MemoryDB + +from git.compat import defenc, force_bytes +from git.cmd import Git +import git.diff as git_diff +from git.exc import CheckoutError, GitCommandError, GitError, InvalidGitRepositoryError +from git.objects import Blob, Commit, Object, Submodule, Tree from git.objects.util import Serializable from git.util import ( + Actor, LazyMixin, LockedFD, join_path_native, file_contents_ro, + _is_path_rooted, + _to_relative_path, to_native_path_linux, unbare_repo, to_bin_sha, ) -from gitdb.base import IStream -from gitdb.db import MemoryDB - -import git.diff as git_diff -import os.path as osp from .fun import ( + S_IFGITLINK, + aggressive_tree_merge, entry_key, - write_cache, read_cache, - aggressive_tree_merge, - write_tree_from_cache, - stat_mode_to_index_mode, - S_IFGITLINK, run_commit_hook, + stat_mode_to_index_mode, + write_cache, + write_tree_from_cache, ) -from .typ import ( - BaseIndexEntry, - IndexEntry, - StageType, -) +from .typ import BaseIndexEntry, IndexEntry, StageType from .util import TemporaryFileSwap, post_clear_cache, default_index, git_working_dir # typing ----------------------------------------------------------------------------- @@ -66,6 +60,7 @@ Any, BinaryIO, Callable, + cast, Dict, Generator, IO, @@ -76,17 +71,16 @@ Sequence, TYPE_CHECKING, Tuple, - Type, Union, ) -from git.types import Commit_ish, PathLike +from git.types import Literal, PathLike if TYPE_CHECKING: from subprocess import Popen - from git.repo import Repo + from git.refs.reference import Reference - from git.util import Actor + from git.repo import Repo Treeish = Union[Tree, Commit, str, bytes] @@ -94,19 +88,18 @@ # ------------------------------------------------------------------------------------ -__all__ = ("IndexFile", "CheckoutError", "StageType") - - @contextlib.contextmanager def _named_temporary_file_for_subprocess(directory: PathLike) -> Generator[str, None, None]: """Create a named temporary file git subprocesses can open, deleting it afterward. - :param directory: The directory in which the file is created. + :param directory: + The directory in which the file is created. - :return: A context manager object that creates the file and provides its name on - entry, and deletes it on exit. + :return: + A context manager object that creates the file and provides its name on entry, + and deletes it on exit. """ - if os.name == "nt": + if sys.platform == "win32": fd, name = tempfile.mkstemp(dir=directory) os.close(fd) try: @@ -123,23 +116,26 @@ class IndexFile(LazyMixin, git_diff.Diffable, Serializable): git command function calls wherever possible. This provides custom merging facilities allowing to merge without actually changing - your index or your working tree. This way you can perform own test-merges based - on the index only without having to deal with the working copy. This is useful - in case of partial working trees. + your index or your working tree. This way you can perform your own test merges based + on the index only without having to deal with the working copy. This is useful in + case of partial working trees. - ``Entries`` + Entries: - The index contains an entries dict whose keys are tuples of type IndexEntry - to facilitate access. + The index contains an entries dict whose keys are tuples of type + :class:`~git.index.typ.IndexEntry` to facilitate access. - You may read the entries dict or manipulate it using IndexEntry instance, i.e.:: + You may read the entries dict or manipulate it using IndexEntry instance, i.e.:: - index.entries[index.entry_key(index_entry_instance)] = index_entry_instance + index.entries[index.entry_key(index_entry_instance)] = index_entry_instance - Make sure you use index.write() once you are done manipulating the index directly - before operating on it using the git command. + Make sure you use :meth:`index.write() ` once you are done manipulating the + index directly before operating on it using the git command. """ + unsafe_git_checkout_index_options = ["--prefix"] + unsafe_git_read_tree_options = ["--index-output"] + __slots__ = ("repo", "version", "entries", "_extension_data", "_file_path") _VERSION = 2 @@ -149,9 +145,9 @@ class IndexFile(LazyMixin, git_diff.Diffable, Serializable): """Flags for a submodule.""" def __init__(self, repo: "Repo", file_path: Union[PathLike, None] = None) -> None: - """Initialize this Index instance, optionally from the given ``file_path``. + """Initialize this Index instance, optionally from the given `file_path`. - If no file_path is given, we will be created from the current index file. + If no `file_path` is given, we will be created from the current index file. If a stream is not given, the stream will be initialized from the current repository's index on demand. @@ -209,7 +205,7 @@ def _deserialize(self, stream: IO) -> "IndexFile": return self def _entries_sorted(self) -> List[IndexEntry]: - """:return: list of entries, in a sorted fashion, first by path, then by stage""" + """:return: List of entries, in a sorted fashion, first by path, then by stage""" return sorted(self.entries.values(), key=lambda e: (e.path, e.stage)) def _serialize(self, stream: IO, ignore_extension_data: bool = False) -> "IndexFile": @@ -230,26 +226,23 @@ def write( """Write the current state to our file path or to the given one. :param file_path: - If None, we will write to our stored file path from which we have - been initialized. Otherwise we write to the given file path. - Please note that this will change the file_path of this index to - the one you gave. + If ``None``, we will write to our stored file path from which we have been + initialized. Otherwise we write to the given file path. Please note that + this will change the `file_path` of this index to the one you gave. :param ignore_extension_data: - If True, the TREE type extension data read in the index will not - be written to disk. NOTE that no extension data is actually written. - Use this if you have altered the index and - would like to use git-write-tree afterwards to create a tree - representing your written changes. - If this data is present in the written index, git-write-tree - will instead write the stored/cached tree. - Alternatively, use IndexFile.write_tree() to handle this case - automatically. + If ``True``, the TREE type extension data read in the index will not be + written to disk. NOTE that no extension data is actually written. Use this + if you have altered the index and would like to use + :manpage:`git-write-tree(1)` afterwards to create a tree representing your + written changes. If this data is present in the written index, + :manpage:`git-write-tree(1)` will instead write the stored/cached tree. + Alternatively, use :meth:`write_tree` to handle this case automatically. """ # Make sure we have our entries read before getting a write lock. - # Otherwise it would be done when streaming. This can happen if one - # doesn't change the index, but writes it right away. - self.entries + # Otherwise it would be done when streaming. + # This can happen if one doesn't change the index, but writes it right away. + self.entries # noqa: B018 lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) @@ -267,30 +260,44 @@ def write( @post_clear_cache @default_index - def merge_tree(self, rhs: Treeish, base: Union[None, Treeish] = None) -> "IndexFile": - """Merge the given rhs treeish into the current index, possibly taking + def merge_tree( + self, + rhs: Treeish, + base: Union[None, Treeish] = None, + allow_unsafe_options: bool = False, + ) -> "IndexFile": + """Merge the given `rhs` treeish into the current index, possibly taking a common base treeish into account. - As opposed to the :func:`IndexFile.from_tree` method, this allows you to use an - already existing tree as the left side of the merge. + As opposed to the :func:`from_tree` method, this allows you to use an already + existing tree as the left side of the merge. :param rhs: Treeish reference pointing to the 'other' side of the merge. :param base: - Optional treeish reference pointing to the common base of 'rhs' and this + Optional treeish reference pointing to the common base of `rhs` and this index which equals lhs. + :param allow_unsafe_options: + Allow options that may write to arbitrary paths. + :return: self (containing the merge and possibly unmerged entries in case of conflicts) - :raise GitCommandError: + :raise git.exc.GitCommandError: If there is a merge conflict. The error will be raised at the first conflicting path. If you want to have proper merge resolution to be done by yourself, you have to commit the changed index (or make a valid tree from - it) and retry with a three-way index.from_tree call. + it) and retry with a three-way :meth:`index.from_tree ` call. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([base, rhs]), + unsafe_options=self.unsafe_git_read_tree_options, + ) + # -i : ignore working tree status # --aggressive : handle more merge cases # -m : do an actual merge @@ -308,15 +315,16 @@ def new(cls, repo: "Repo", *tree_sha: Union[str, Tree]) -> "IndexFile": This method behaves like ``git-read-tree --aggressive`` when doing the merge. - :param repo: The repository treeish are located in. + :param repo: + The repository treeish are located in. :param tree_sha: 20 byte or 40 byte tree sha or tree objects. :return: - New IndexFile instance. Its path will be undefined. - If you intend to write such a merged Index, supply an alternate file_path - to its 'write' method. + New :class:`IndexFile` instance. Its path will be undefined. + If you intend to write such a merged Index, supply an alternate + ``file_path`` to its :meth:`write` method. """ tree_sha_bytes: List[bytes] = [to_bin_sha(str(t)) for t in tree_sha] base_entries = aggressive_tree_merge(repo.odb, tree_sha_bytes) @@ -334,44 +342,62 @@ def new(cls, repo: "Repo", *tree_sha: Union[str, Tree]) -> "IndexFile": return inst @classmethod - def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile": - """Merge the given treeish revisions into a new index which is returned. + def from_tree( + cls, + repo: "Repo", + *treeish: Treeish, + allow_unsafe_options: bool = False, + **kwargs: Any, + ) -> "IndexFile": + R"""Merge the given treeish revisions into a new index which is returned. The original index will remain unaltered. :param repo: The repository treeish are located in. :param treeish: - One, two or three Tree Objects, Commits or 40 byte hexshas. The result - changes according to the amount of trees. - If 1 Tree is given, it will just be read into a new index - If 2 Trees are given, they will be merged into a new index using a - two way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other' - one. It behaves like a fast-forward. - If 3 Trees are given, a 3-way merge will be performed with the first tree - being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' tree, - tree 3 is the 'other' one. + One, two or three :class:`~git.objects.tree.Tree` objects, + :class:`~git.objects.commit.Commit`\s or 40 byte hexshas. + + The result changes according to the amount of trees: + + 1. If 1 Tree is given, it will just be read into a new index. + 2. If 2 Trees are given, they will be merged into a new index using a two + way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other' + one. It behaves like a fast-forward. + 3. If 3 Trees are given, a 3-way merge will be performed with the first tree + being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' + tree, tree 3 is the 'other' one. :param kwargs: - Additional arguments passed to git-read-tree. + Additional arguments passed to :manpage:`git-read-tree(1)`. + + :param allow_unsafe_options: + Allow options that may write to arbitrary paths. :return: - New IndexFile instance. It will point to a temporary index location which - does not exist anymore. If you intend to write such a merged Index, supply - an alternate file_path to its 'write' method. + New :class:`IndexFile` instance. It will point to a temporary index location + which does not exist anymore. If you intend to write such a merged Index, + supply an alternate ``file_path`` to its :meth:`write` method. :note: - In the three-way merge case, --aggressive will be specified to automatically - resolve more cases in a commonly correct manner. Specify trivial=True as kwarg - to override that. + In the three-way merge case, ``--aggressive`` will be specified to + automatically resolve more cases in a commonly correct manner. Specify + ``trivial=True`` as a keyword argument to override that. - As the underlying git-read-tree command takes into account the current - index, it will be temporarily moved out of the way to prevent any unexpected - interference. + As the underlying :manpage:`git-read-tree(1)` command takes into account the + current index, it will be temporarily moved out of the way to prevent any + unexpected interference. """ if len(treeish) == 0 or len(treeish) > 3: raise ValueError("Please specify between 1 and 3 treeish, got %i" % len(treeish)) + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates(treeish, kwargs), + unsafe_options=cls.unsafe_git_read_tree_options, + ) + arg_list: List[Union[Treeish, str]] = [] # Ignore that the working tree and index possibly are out of date. if len(treeish) > 1: @@ -387,14 +413,14 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile arg_list.append("--index-output=%s" % tmp_index) arg_list.extend(treeish) - # Move the current index out of the way - otherwise the merge may fail - # as it considers existing entries. Moving it essentially clears the index. + # Move the current index out of the way - otherwise the merge may fail as it + # considers existing entries. Moving it essentially clears the index. # Unfortunately there is no 'soft' way to do it. # The TemporaryFileSwap ensures the original file gets put back. with TemporaryFileSwap(join_path_native(repo.git_dir, "index")): repo.git.read_tree(*arg_list, **kwargs) index = cls(repo, tmp_index) - index.entries # Force it to read the file as we will delete the temp-file. + index.entries # noqa: B018 # Force it to read the file as we will delete the temp-file. return index # END index merge handling @@ -402,12 +428,13 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile @unbare_repo def _iter_expand_paths(self: "IndexFile", paths: Sequence[PathLike]) -> Iterator[PathLike]: - """Expand the directories in list of paths to the corresponding paths accordingly. + """Expand the directories in list of paths to the corresponding paths + accordingly. :note: - git will add items multiple times even if a glob overlapped - with manually specified paths or if paths where specified multiple - times - we respect that and do not prune. + git will add items multiple times even if a glob overlapped with manually + specified paths or if paths where specified multiple times - we respect that + and do not prune. """ def raise_exc(e: Exception) -> NoReturn: @@ -416,7 +443,7 @@ def raise_exc(e: Exception) -> NoReturn: r = str(self.repo.working_tree_dir) rs = r + os.sep for path in paths: - abs_path = str(path) + abs_path = os.fspath(path) if not osp.isabs(abs_path): abs_path = osp.join(r, path) # END make absolute path @@ -436,11 +463,11 @@ def raise_exc(e: Exception) -> NoReturn: if not os.path.exists(abs_path) and ("?" in abs_path or "*" in abs_path or "[" in abs_path): resolved_paths = glob.glob(abs_path) # not abs_path in resolved_paths: - # a glob() resolving to the same path we are feeding it with - # is a glob() that failed to resolve. If we continued calling - # ourselves we'd endlessly recurse. If the condition below - # evaluates to true then we are likely dealing with a file - # whose name contains wildcard characters. + # A glob() resolving to the same path we are feeding it with is a + # glob() that failed to resolve. If we continued calling ourselves + # we'd endlessly recurse. If the condition below evaluates to true + # then we are likely dealing with a file whose name contains wildcard + # characters. if abs_path not in resolved_paths: for f in self._iter_expand_paths(glob.glob(abs_path)): yield str(f).replace(rs, "") @@ -468,22 +495,27 @@ def _write_path_to_stdin( fprogress: Callable[[PathLike, bool, PathLike], None], read_from_stdout: bool = True, ) -> Union[None, str]: - """Write path to proc.stdin and make sure it processes the item, including progress. + """Write path to ``proc.stdin`` and make sure it processes the item, including + progress. - :return: stdout string + :return: + stdout string - :param read_from_stdout: if True, proc.stdout will be read after the item - was sent to stdin. In that case, it will return None. + :param read_from_stdout: + If ``True``, ``proc.stdout`` will be read after the item was sent to stdin. + In that case, it will return ``None``. - :note: There is a bug in git-update-index that prevents it from sending - reports just in time. This is why we have a version that tries to - read stdout and one which doesn't. In fact, the stdout is not - important as the piped-in files are processed anyway and just in time. + :note: + There is a bug in :manpage:`git-update-index(1)` that prevents it from + sending reports just in time. This is why we have a version that tries to + read stdout and one which doesn't. In fact, the stdout is not important as + the piped-in files are processed anyway and just in time. - :note: Newlines are essential here, gits behaviour is somewhat inconsistent - on this depending on the version, hence we try our best to deal with - newlines carefully. Usually the last newline will not be sent, instead - we will close stdin to break the pipe. + :note: + Newlines are essential here, git's behaviour is somewhat inconsistent on + this depending on the version, hence we try our best to deal with newlines + carefully. Usually the last newline will not be sent, instead we will close + stdin to break the pipe. """ fprogress(filepath, False, item) rval: Union[None, str] = None @@ -506,12 +538,14 @@ def iter_blobs( self, predicate: Callable[[Tuple[StageType, Blob]], bool] = lambda t: True ) -> Iterator[Tuple[StageType, Blob]]: """ - :return: Iterator yielding tuples of Blob objects and stages, tuple(stage, Blob) + :return: + Iterator yielding tuples of :class:`~git.objects.blob.Blob` objects and + stages, tuple(stage, Blob). :param predicate: - Function(t) returning True if tuple(stage, Blob) should be yielded by the - iterator. A default filter, the BlobFilter, allows you to yield blobs - only if they match a given list of paths. + Function(t) returning ``True`` if tuple(stage, Blob) should be yielded by + the iterator. A default filter, the :class:`~git.index.typ.BlobFilter`, allows you + to yield blobs only if they match a given list of paths. """ for entry in self.entries.values(): blob = entry.to_blob(self.repo) @@ -524,16 +558,18 @@ def iter_blobs( def unmerged_blobs(self) -> Dict[PathLike, List[Tuple[StageType, Blob]]]: """ :return: - Dict(path : list( tuple( stage, Blob, ...))), being - a dictionary associating a path in the index with a list containing - sorted stage/blob pairs. + Dict(path : list(tuple(stage, Blob, ...))), being a dictionary associating a + path in the index with a list containing sorted stage/blob pairs. :note: - Blobs that have been removed in one side simply do not exist in the - given stage. I.e. a file removed on the 'other' branch whose entries - are at stage 3 will not have a stage 3 entry. + Blobs that have been removed in one side simply do not exist in the given + stage. That is, a file removed on the 'other' branch whose entries are at + stage 3 will not have a stage 3 entry. """ - is_unmerged_blob = lambda t: t[0] != 0 + + def is_unmerged_blob(t: Tuple[StageType, Blob]) -> bool: + return t[0] != 0 + path_map: Dict[PathLike, List[Tuple[StageType, Blob]]] = {} for stage, blob in self.iter_blobs(is_unmerged_blob): path_map.setdefault(blob.path, []).append((stage, blob)) @@ -553,12 +589,14 @@ def resolve_blobs(self, iter_blobs: Iterator[Blob]) -> "IndexFile": This will effectively remove the index entries of the respective path at all non-null stages and add the given blob as new stage null blob. - For each path there may only be one blob, otherwise a ValueError will be raised - claiming the path is already at stage 0. + For each path there may only be one blob, otherwise a :exc:`ValueError` will be + raised claiming the path is already at stage 0. - :raise ValueError: if one of the blobs already existed at stage 0 + :raise ValueError: + If one of the blobs already existed at stage 0. - :return: self + :return: + self :note: You will have to write the index manually once you are done, i.e. @@ -588,28 +626,33 @@ def update(self) -> "IndexFile": """Reread the contents of our index file, discarding all cached information we might have. - :note: This is a possibly dangerous operations as it will discard your changes - to index.entries. + :note: + This is a possibly dangerous operations as it will discard your changes to + :attr:`index.entries `. - :return: self + :return: + self """ self._delete_entries_cache() # Allows to lazily reread on demand. return self def write_tree(self) -> Tree: - """Write this index to a corresponding Tree object into the repository's - object database and return it. + """Write this index to a corresponding :class:`~git.objects.tree.Tree` object + into the repository's object database and return it. - :return: Tree object representing this index. + :return: + :class:`~git.objects.tree.Tree` object representing this index. - :note: The tree will be written even if one or more objects the tree refers to - does not yet exist in the object database. This could happen if you added - Entries to the index directly. + :note: + The tree will be written even if one or more objects the tree refers to does + not yet exist in the object database. This could happen if you added entries + to the index directly. - :raise ValueError: if there are no entries in the cache + :raise ValueError: + If there are no entries in the cache. - :raise UnmergedEntriesError: + :raise git.exc.UnmergedEntriesError: """ # We obtain no lock as we just flush our contents to disk as tree. # If we are a new index, the entries access will load our data accordingly. @@ -620,16 +663,16 @@ def write_tree(self) -> Tree: # Copy changed trees only. mdb.stream_copy(mdb.sha_iter(), self.repo.odb) - # Note: Additional deserialization could be saved if write_tree_from_cache - # would return sorted tree entries. + # Note: Additional deserialization could be saved if write_tree_from_cache would + # return sorted tree entries. root_tree = Tree(self.repo, binsha, path="") root_tree._cache = tree_items return root_tree def _process_diff_args( - self, # type: ignore[override] - args: List[Union[str, "git_diff.Diffable", Type["git_diff.Diffable.Index"]]], - ) -> List[Union[str, "git_diff.Diffable", Type["git_diff.Diffable.Index"]]]: + self, + args: List[Union[PathLike, "git_diff.Diffable"]], + ) -> List[Union[PathLike, "git_diff.Diffable"]]: try: args.pop(args.index(self)) except IndexError: @@ -639,18 +682,21 @@ def _process_diff_args( def _to_relative_path(self, path: PathLike) -> PathLike: """ - :return: Version of path relative to our git directory or raise ValueError - if it is not within our git directory""" - if not osp.isabs(path): - return path + :return: + Version of path relative to our git directory or raise :exc:`ValueError` if + it is not within our git directory. + + :raise ValueError: + """ if self.repo.bare: - raise InvalidGitRepositoryError("require non-bare repository") - if not str(path).startswith(str(self.repo.working_tree_dir)): - raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) - return os.path.relpath(path, self.repo.working_tree_dir) + drive, _tail = osp.splitdrive(os.fspath(path)) + if drive or _is_path_rooted(path): + raise InvalidGitRepositoryError("paths with a drive or root require a non-bare repository") + return path + return _to_relative_path(cast(PathLike, self.repo.working_tree_dir), path) def _preprocess_add_items( - self, items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]] + self, items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]] ) -> Tuple[List[PathLike], List[BaseIndexEntry]]: """Split the items into two lists of path strings and BaseEntries.""" paths = [] @@ -672,15 +718,24 @@ def _preprocess_add_items( return paths, entries def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry: - """Store file at filepath in the database and return the base index entry - Needs the git_working_dir decorator active ! This must be assured in the calling code""" + """Store file at filepath in the database and return the base index entry. + + :note: + This needs the :func:`~git.index.util.git_working_dir` decorator active! + This must be ensured in the calling code. + """ st = os.lstat(filepath) # Handles non-symlinks as well. + if S_ISLNK(st.st_mode): # In PY3, readlink is a string, but we need bytes. # In PY2, it was just OS encoded bytes, we assumed UTF-8. - open_stream: Callable[[], BinaryIO] = lambda: BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) + def open_stream() -> BinaryIO: + return BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) else: - open_stream = lambda: open(filepath, "rb") + + def open_stream() -> BinaryIO: + return open(filepath, "rb") + with open_stream() as stream: fprogress(filepath, False, filepath) istream = self.repo.odb.store(IStream(Blob.type, st.st_size, stream)) @@ -705,14 +760,17 @@ def _entries_for_paths( ) -> List[BaseIndexEntry]: entries_added: List[BaseIndexEntry] = [] if path_rewriter: + working_tree_dir = self.repo.working_tree_dir + if working_tree_dir is None: + raise InvalidGitRepositoryError("Cannot rewrite paths without a working tree") + working_tree_dir = str(working_tree_dir) for path in paths: if osp.isabs(path): abspath = path - gitrelative_path = path[len(str(self.repo.working_tree_dir)) + 1 :] + gitrelative_path = path[len(working_tree_dir) + 1 :] else: gitrelative_path = path - if self.repo.working_tree_dir: - abspath = osp.join(self.repo.working_tree_dir, gitrelative_path) + abspath = osp.join(working_tree_dir, gitrelative_path) # END obtain relative and absolute paths blob = Blob( @@ -737,15 +795,15 @@ def _entries_for_paths( def add( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], force: bool = True, fprogress: Callable = lambda *args: None, path_rewriter: Union[Callable[..., PathLike], None] = None, write: bool = True, write_extension_data: bool = False, ) -> List[BaseIndexEntry]: - """Add files from the working tree, specific blobs or BaseIndexEntries - to the index. + R"""Add files from the working tree, specific blobs, or + :class:`~git.index.typ.BaseIndexEntry`\s to the index. :param items: Multiple types of items are supported, types can be mixed within one call. @@ -753,9 +811,10 @@ def add( relative or absolute. - path string + Strings denote a relative or absolute path into the repository pointing - to an existing file, e.g., CHANGES, lib/myfile.ext, - '/home/gitrepo/lib/myfile.ext'. + to an existing file, e.g., ``CHANGES``, ``lib/myfile.ext``, + ``/home/gitrepo/lib/myfile.ext``. Absolute paths must start with working tree directory of this index's repository to be considered valid. For example, if it was initialized @@ -769,11 +828,13 @@ def add( directories like ``lib``, which will add all the files within the directory and subdirectories. - This equals a straight git-add. + This equals a straight :manpage:`git-add(1)`. They are added at stage 0. - - Blob or Submodule object + - :class:`~git.objects.blob.Blob` or + :class:`~git.objects.submodule.base.Submodule` object + Blobs are added as they are assuming a valid mode is set. The file they refer to may or may not exist in the file system, but must @@ -786,75 +847,84 @@ def add( The handling now very much equals the way string paths are processed, except that the mode you have set will be kept. This allows you to create symlinks by settings the mode respectively and writing the target - of the symlink directly into the file. This equals a default - Linux-Symlink which is not dereferenced automatically, except that it - can be created on filesystems not supporting it as well. + of the symlink directly into the file. This equals a default Linux + symlink which is not dereferenced automatically, except that it can be + created on filesystems not supporting it as well. - Please note that globs or directories are not allowed in Blob objects. + Please note that globs or directories are not allowed in + :class:`~git.objects.blob.Blob` objects. They are added at stage 0. - - BaseIndexEntry or type - Handling equals the one of Blob objects, but the stage may be explicitly - set. Please note that Index Entries require binary sha's. + - :class:`~git.index.typ.BaseIndexEntry` or type + + Handling equals the one of :class:`~git.objects.blob.Blob` objects, but + the stage may be explicitly set. Please note that Index Entries require + binary sha's. :param force: **CURRENTLY INEFFECTIVE** - If True, otherwise ignored or excluded files will be added anyway. - As opposed to the git-add command, we enable this flag by default - as the API user usually wants the item to be added even though - they might be excluded. + If ``True``, otherwise ignored or excluded files will be added anyway. As + opposed to the :manpage:`git-add(1)` command, we enable this flag by default + as the API user usually wants the item to be added even though they might be + excluded. :param fprogress: - Function with signature f(path, done=False, item=item) called for each - path to be added, one time once it is about to be added where done==False - and once after it was added where done=True. - item is set to the actual item we handle, either a Path or a BaseIndexEntry - Please note that the processed path is not guaranteed to be present - in the index already as the index is currently being processed. + Function with signature ``f(path, done=False, item=item)`` called for each + path to be added, one time once it is about to be added where ``done=False`` + and once after it was added where ``done=True``. + + ``item`` is set to the actual item we handle, either a path or a + :class:`~git.index.typ.BaseIndexEntry`. + + Please note that the processed path is not guaranteed to be present in the + index already as the index is currently being processed. :param path_rewriter: - Function with signature (string) func(BaseIndexEntry) function returning a - path for each passed entry which is the path to be actually recorded for the - object created from entry.path. This allows you to write an index which is - not identical to the layout of the actual files on your hard-disk. If not - None and ``items`` contain plain paths, these paths will be converted to - Entries beforehand and passed to the path_rewriter. Please note that - entry.path is relative to the git repository. + Function, with signature ``(string) func(BaseIndexEntry)``, returning a path + for each passed entry which is the path to be actually recorded for the + object created from :attr:`entry.path `. + This allows you to write an index which is not identical to the layout of + the actual files on your hard-disk. If not ``None`` and `items` contain + plain paths, these paths will be converted to Entries beforehand and passed + to the path_rewriter. Please note that ``entry.path`` is relative to the git + repository. :param write: - If True, the index will be written once it was altered. Otherwise - the changes only exist in memory and are not available to git commands. + If ``True``, the index will be written once it was altered. Otherwise the + changes only exist in memory and are not available to git commands. :param write_extension_data: - If True, extension data will be written back to the index. This can lead to - issues in case it is containing the 'TREE' extension, which will cause the - `git commit` command to write an old tree, instead of a new one representing - the now changed index. + If ``True``, extension data will be written back to the index. This can lead + to issues in case it is containing the 'TREE' extension, which will cause + the :manpage:`git-commit(1)` command to write an old tree, instead of a new + one representing the now changed index. This doesn't matter if you use :meth:`IndexFile.commit`, which ignores the - `TREE` extension altogether. You should set it to True if you intend to use - :meth:`IndexFile.commit` exclusively while maintaining support for + 'TREE' extension altogether. You should set it to ``True`` if you intend to + use :meth:`IndexFile.commit` exclusively while maintaining support for third-party extensions. Besides that, you can usually safely ignore the built-in extensions when using GitPython on repositories that are not handled manually at all. All current built-in extensions are listed here: - http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/technical/index-format.txt + https://git-scm.com/docs/index-format :return: - List(BaseIndexEntries) representing the entries just actually added. + List of :class:`~git.index.typ.BaseIndexEntry`\s representing the entries + just actually added. :raise OSError: - If a supplied Path did not exist. Please note that BaseIndexEntry - Objects that do not have a null sha will be added even if their paths - do not exist. + If a supplied path did not exist. Please note that + :class:`~git.index.typ.BaseIndexEntry` objects that do not have a null sha + will be added even if their paths do not exist. """ - # Sort the entries into strings and Entries. Blobs are converted to entries automatically. + # Sort the entries into strings and Entries. + # Blobs are converted to entries automatically. # Paths can be git-added. For everything else we use git-update-index. paths, entries = self._preprocess_add_items(items) entries_added: List[BaseIndexEntry] = [] - # This code needs a working tree, therefore we try not to run it unless required. + # This code needs a working tree, so we try not to run it unless required. # That way, we are OK on a bare repository as well. # If there are no paths, the rewriter has nothing to do either. if paths: @@ -897,7 +967,8 @@ def handle_null_entries(self: "IndexFile") -> None: # END null_entry handling # REWRITE PATHS - # If we have to rewrite the entries, do so now, after we have generated all object sha's. + # If we have to rewrite the entries, do so now, after we have generated all + # object sha's. if path_rewriter: for i, e in enumerate(entries): entries[i] = BaseIndexEntry((e.mode, e.binsha, e.stage, path_rewriter(e))) @@ -951,53 +1022,66 @@ def _items_to_rela_paths( @default_index def remove( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], working_tree: bool = False, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> List[str]: - """Remove the given items from the index and optionally from - the working tree as well. + R"""Remove the given items from the index and optionally from the working tree + as well. :param items: Multiple types of items are supported which may be be freely mixed. - path string + Remove the given path at all stages. If it is a directory, you must - specify the r=True keyword argument to remove all file entries - below it. If absolute paths are given, they will be converted - to a path relative to the git repository directory containing - the working tree + specify the ``r=True`` keyword argument to remove all file entries below + it. If absolute paths are given, they will be converted to a path + relative to the git repository directory containing the working tree + + The path string may include globs, such as ``*.c``. - The path string may include globs, such as \\*.c. + - :class:`~git.objects.blob.Blob` object - - Blob Object Only the path portion is used in this case. - - BaseIndexEntry or compatible type - The only relevant information here Yis the path. The stage is ignored. + - :class:`~git.index.typ.BaseIndexEntry` or compatible type + + The only relevant information here is the path. The stage is ignored. :param working_tree: - If True, the entry will also be removed from the working tree, physically - removing the respective file. This may fail if there are uncommitted changes - in it. + If ``True``, the entry will also be removed from the working tree, + physically removing the respective file. This may fail if there are + uncommitted changes in it. + + :param allow_unsafe_options: + Allow unsafe options such as ``--pathspec-from-file`` to be passed to + :manpage:`git-rm(1)`. :param kwargs: - Additional keyword arguments to be passed to git-rm, such - as 'r' to allow recursive removal of + Additional keyword arguments to be passed to :manpage:`git-rm(1)`, such as + ``r`` to allow recursive removal. :return: - List(path_string, ...) list of repository relative paths that have - been removed effectively. - This is interesting to know in case you have provided a directory or - globs. Paths are relative to the repository. + List(path_string, ...) list of repository relative paths that have been + removed effectively. + + This is interesting to know in case you have provided a directory or globs. + Paths are relative to the repository. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=Git.unsafe_git_pathspec_from_file_options, + ) args = [] if not working_tree: args.append("--cached") args.append("--") # Preprocess paths. - paths = self._items_to_rela_paths(items) + paths = list(map(os.fspath, self._items_to_rela_paths(items))) # type: ignore[arg-type] removed_paths = self.repo.git.rm(args, paths, **kwargs).splitlines() # Process output to gain proper paths. @@ -1008,33 +1092,43 @@ def remove( @default_index def move( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], skip_errors: bool = False, **kwargs: Any, ) -> List[Tuple[str, str]]: """Rename/move the items, whereas the last item is considered the destination of - the move operation. If the destination is a file, the first item (of two) - must be a file as well. If the destination is a directory, it may be preceded - by one or more directories or files. + the move operation. + + If the destination is a file, the first item (of two) must be a file as well. + + If the destination is a directory, it may be preceded by one or more directories + or files. The working tree will be affected in non-bare repositories. - :parma items: + :param items: Multiple types of items are supported, please see the :meth:`remove` method for reference. + :param skip_errors: - If True, errors such as ones resulting from missing source files will - be skipped. + If ``True``, errors such as ones resulting from missing source files will be + skipped. + :param kwargs: - Additional arguments you would like to pass to git-mv, such as dry_run - or force. + Additional arguments you would like to pass to :manpage:`git-mv(1)`, such as + ``dry_run`` or ``force``. + + :return: + List(tuple(source_path_string, destination_path_string), ...) - :return: List(tuple(source_path_string, destination_path_string), ...) - A list of pairs, containing the source file moved as well as its - actual destination. Relative to the repository root. + A list of pairs, containing the source file moved as well as its actual + destination. Relative to the repository root. - :raise ValueError: If only one item was given - :raise GitCommandError: If git could not handle your request + :raise ValueError: + If only one item was given. + + :raise git.exc.GitCommandError: + If git could not handle your request. """ args = [] if skip_errors: @@ -1047,13 +1141,13 @@ def move( was_dry_run = kwargs.pop("dry_run", kwargs.pop("n", None)) kwargs["dry_run"] = True - # First execute rename in dryrun so the command tells us what it actually does + # First execute rename in dry run so the command tells us what it actually does # (for later output). out = [] mvlines = self.repo.git.mv(args, paths, **kwargs).splitlines() - # Parse result - first 0:n/2 lines are 'checking ', the remaining ones - # are the 'renaming' ones which we parse. + # Parse result - first 0:n/2 lines are 'checking ', the remaining ones are the + # 'renaming' ones which we parse. for ln in range(int(len(mvlines) / 2), len(mvlines)): tokens = mvlines[ln].split(" to ") assert len(tokens) == 2, "Too many tokens in %s" % mvlines[ln] @@ -1066,7 +1160,7 @@ def move( # Either prepare for the real run, or output the dry-run result. if was_dry_run: return out - # END handle dryrun + # END handle dry run # Now apply the actual operation. kwargs.pop("dry_run") @@ -1077,25 +1171,31 @@ def move( def commit( self, message: str, - parent_commits: Union[Commit_ish, None] = None, + parent_commits: Union[List[Commit], None] = None, head: bool = True, - author: Union[None, "Actor"] = None, - committer: Union[None, "Actor"] = None, + author: Union[None, Actor] = None, + committer: Union[None, Actor] = None, author_date: Union[datetime.datetime, str, None] = None, commit_date: Union[datetime.datetime, str, None] = None, skip_hooks: bool = False, + trailers: Union[None, "Dict[str, str]", "List[Tuple[str, str]]"] = None, ) -> Commit: - """Commit the current default index file, creating a Commit object. + """Commit the current default index file, creating a + :class:`~git.objects.commit.Commit` object. For more information on the arguments, see :meth:`Commit.create_from_tree `. - :note: If you have manually altered the :attr:`entries` member of this instance, - don't forget to :meth:`write` your changes to disk beforehand. - Passing ``skip_hooks=True`` is the equivalent of using ``-n`` - or ``--no-verify`` on the command line. + :note: + If you have manually altered the :attr:`entries` member of this instance, + don't forget to :meth:`write` your changes to disk beforehand. + + :note: + Passing ``skip_hooks=True`` is the equivalent of using ``-n`` or + ``--no-verify`` on the command line. - :return: :class:`Commit` object representing the new commit + :return: + :class:`~git.objects.commit.Commit` object representing the new commit """ if not skip_hooks: run_commit_hook("pre-commit", self) @@ -1115,6 +1215,7 @@ def commit( committer=committer, author_date=author_date, commit_date=commit_date, + trailers=trailers, ) if not skip_hooks: run_commit_hook("post-commit", self) @@ -1155,50 +1256,65 @@ def checkout( paths: Union[None, Iterable[PathLike]] = None, force: bool = False, fprogress: Callable = lambda *args: None, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> Union[None, Iterator[PathLike], Sequence[PathLike]]: """Check out the given paths or all files from the version known to the index into the working tree. - :note: Be sure you have written pending changes using the :meth:`write` method - in case you have altered the entries dictionary directly. + :note: + Be sure you have written pending changes using the :meth:`write` method in + case you have altered the entries dictionary directly. :param paths: - If None, all paths in the index will be checked out. Otherwise an iterable - of relative or absolute paths or a single path pointing to files or directories - in the index is expected. + If ``None``, all paths in the index will be checked out. + Otherwise an iterable of relative or absolute paths or a single path + pointing to files or directories in the index is expected. :param force: - If True, existing files will be overwritten even if they contain local modifications. - If False, these will trigger a :class:`CheckoutError`. + If ``True``, existing files will be overwritten even if they contain local + modifications. + If ``False``, these will trigger a :exc:`~git.exc.CheckoutError`. :param fprogress: - see :func:`IndexFile.add` for signature and explanation. - The provided progress information will contain None as path and item if no - explicit paths are given. Otherwise progress information will be send + See :meth:`IndexFile.add` for signature and explanation. + + The provided progress information will contain ``None`` as path and item if + no explicit paths are given. Otherwise progress information will be send prior and after a file has been checked out. + :param allow_unsafe_options: + Allow unsafe options, such as ``--prefix``. + :param kwargs: - Additional arguments to be passed to git-checkout-index. + Additional arguments to be passed to :manpage:`git-checkout-index(1)`. :return: - iterable yielding paths to files which have been checked out and are + Iterable yielding paths to files which have been checked out and are guaranteed to match the version stored in the index. - :raise exc.CheckoutError: - If at least one file failed to be checked out. This is a summary, - hence it will checkout as many files as it can anyway. - If one of files or directories do not exist in the index - ( as opposed to the original git command who ignores them ). - Raise GitCommandError if error lines could not be parsed - this truly is - an exceptional state. - - .. note:: The checkout is limited to checking out the files in the - index. Files which are not in the index anymore and exist in - the working tree will not be deleted. This behaviour is fundamentally - different to *head.checkout*, i.e. if you want git-checkout like behaviour, - use head.checkout instead of index.checkout. + :raise git.exc.CheckoutError: + * If at least one file failed to be checked out. This is a summary, hence it + will checkout as many files as it can anyway. + * If one of files or directories do not exist in the index (as opposed to + the original git command, which ignores them). + + :raise git.exc.GitCommandError: + If error lines could not be parsed - this truly is an exceptional state. + + :note: + The checkout is limited to checking out the files in the index. Files which + are not in the index anymore and exist in the working tree will not be + deleted. This behaviour is fundamentally different to ``head.checkout``, + i.e. if you want :manpage:`git-checkout(1)`-like behaviour, use + ``head.checkout`` instead of ``index.checkout``. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_checkout_index_options, + ) + args = ["--index"] if force: args.append("--force") @@ -1276,17 +1392,20 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik if isinstance(paths, str): paths = [paths] - # Make sure we have our entries loaded before we start checkout_index, - # which will hold a lock on it. We try to get the lock as well during - # our entries initialization. - self.entries + # Make sure we have our entries loaded before we start checkout_index, which + # will hold a lock on it. We try to get the lock as well during our entries + # initialization. + self.entries # noqa: B018 args.append("--stdin") kwargs["as_process"] = True kwargs["istream"] = subprocess.PIPE proc = self.repo.git.checkout_index(args, **kwargs) + # FIXME: Reading from GIL! - make_exc = lambda: GitCommandError(("git-checkout-index",) + tuple(args), 128, proc.stderr.read()) + def make_exc() -> GitCommandError: + return GitCommandError(("git-checkout-index", *args), 128, proc.stderr.read()) + checked_out_files: List[PathLike] = [] for path in paths: @@ -1297,11 +1416,11 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik try: self.entries[(co_path, 0)] except KeyError: - folder = str(co_path) + folder = co_path if not folder.endswith("/"): folder += "/" for entry in self.entries.values(): - if str(entry.path).startswith(folder): + if os.fspath(entry.path).startswith(folder): p = entry.path self._write_path_to_stdin(proc, p, p, make_exc, fprogress, read_from_stdout=False) checked_out_files.append(p) @@ -1319,7 +1438,7 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik self._flush_stdin_and_wait(proc, ignore_stdout=True) except GitCommandError: # Without parsing stdout we don't know what failed. - raise CheckoutError( + raise CheckoutError( # noqa: B904 "Some files could not be checked out from the index, probably because they didn't exist.", failed_files, [], @@ -1337,54 +1456,69 @@ def reset( working_tree: bool = False, paths: Union[None, Iterable[PathLike]] = None, head: bool = False, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> "IndexFile": - """Reset the index to reflect the tree at the given commit. This will not - adjust our HEAD reference as opposed to HEAD.reset by default. + """Reset the index to reflect the tree at the given commit. This will not adjust + our HEAD reference by default, as opposed to + :meth:`HEAD.reset `. :param commit: - Revision, Reference or Commit specifying the commit we should represent. - If you want to specify a tree only, use IndexFile.from_tree and overwrite - the default index. + Revision, :class:`~git.refs.reference.Reference` or + :class:`~git.objects.commit.Commit` specifying the commit we should + represent. + + If you want to specify a tree only, use :meth:`IndexFile.from_tree` and + overwrite the default index. :param working_tree: - If True, the files in the working tree will reflect the changed index. - If False, the working tree will not be touched + If ``True``, the files in the working tree will reflect the changed index. + If ``False``, the working tree will not be touched. Please note that changes to the working copy will be discarded without - warning ! + warning! :param head: - If True, the head will be set to the given commit. This is False by default, - but if True, this method behaves like HEAD.reset. + If ``True``, the head will be set to the given commit. This is ``False`` by + default, but if ``True``, this method behaves like + :meth:`HEAD.reset `. - :param paths: if given as an iterable of absolute or repository-relative paths, - only these will be reset to their state at the given commit-ish. + :param paths: + If given as an iterable of absolute or repository-relative paths, only these + will be reset to their state at the given commit-ish. The paths need to exist at the commit, otherwise an exception will be raised. + :param allow_unsafe_options: + Allow options that may write to arbitrary paths. + :param kwargs: - Additional keyword arguments passed to git-reset + Additional keyword arguments passed to :manpage:`git-reset(1)`. - .. note:: IndexFile.reset, as opposed to HEAD.reset, will not delete any files - in order to maintain a consistent working tree. Instead, it will just - check out the files according to their state in the index. - If you want git-reset like behaviour, use *HEAD.reset* instead. + :note: + :meth:`IndexFile.reset`, as opposed to + :meth:`HEAD.reset `, will not delete any files in + order to maintain a consistent working tree. Instead, it will just check out + the files according to their state in the index. + If you want :manpage:`git-reset(1)`-like behaviour, use + :meth:`HEAD.reset ` instead. - :return: self""" - # What we actually want to do is to merge the tree into our existing - # index, which is what git-read-tree does. - new_inst = type(self).from_tree(self.repo, commit) + :return: + self + """ + # What we actually want to do is to merge the tree into our existing index, + # which is what git-read-tree does. + new_inst = type(self).from_tree(self.repo, commit, allow_unsafe_options=allow_unsafe_options) if not paths: self.entries = new_inst.entries else: nie = new_inst.entries for path in paths: path = self._to_relative_path(path) + key = entry_key(path, 0) try: - key = entry_key(path, 0) self.entries[key] = nie[key] except KeyError: - # If key is not in theirs, it musn't be in ours. + # If key is not in theirs, it mustn't be in ours. try: del self.entries[key] except KeyError: @@ -1405,34 +1539,88 @@ def reset( return self - # @ default_index, breaks typing for some reason, copied into function def diff( - self, # type: ignore[override] - other: Union[Type["git_diff.Diffable.Index"], "Tree", "Commit", str, None] = git_diff.Diffable.Index, + self, + other: Union[ + Literal[git_diff.DiffConstants.INDEX], + Literal[git_diff.DiffConstants.NULL_TREE], + "Tree", + "Commit", + str, + None, + ] = git_diff.INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, + allow_unsafe_options: bool = False, **kwargs: Any, - ) -> git_diff.DiffIndex: - """Diff this index against the working copy or a Tree or Commit object. + ) -> git_diff.DiffIndex[git_diff.Diff]: + """Diff this index against the working copy or a :class:`~git.objects.tree.Tree` + or :class:`~git.objects.commit.Commit` object. For documentation of the parameters and return values, see :meth:`Diffable.diff `. :note: - Will only work with indices that represent the default git index as - they have not been initialized with a stream. + Will only work with indices that represent the default git index as they + have not been initialized with a stream. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([other], kwargs), + unsafe_options=self.repo.unsafe_git_diff_options, + clusterable_short_options="46abceflmnpqrstuvwzBCDMNRW", + ) # Only run if we are the default repository index. if self._file_path != self._index_path(): raise AssertionError("Cannot call %r on indices that do not represent the default git index" % self.diff()) # Index against index is always empty. - if other is self.Index: + if other is self.INDEX: return git_diff.DiffIndex() - # Index against anything but None is a reverse diff with the respective - # item. Handle existing -R flags properly. Transform strings to the object - # so that we can call diff on it. + if other == git_diff.NULL_TREE or other == git_diff.NULL_TREE_SHA: + args: List[Union[PathLike, str]] = [ + "--cached", + git_diff.NULL_TREE_SHA, + "--abbrev=40", + "--full-index", + ] + + if not any(x in kwargs for x in ("find_renames", "no_renames", "M")): + args.append("-M") + + if create_patch: + args.append("-p") + args.append("--no-ext-diff") + else: + args.append("--raw") + args.append("-z") + + args.append("--no-color") + + if paths is not None and not isinstance(paths, (tuple, list)): + paths = [paths] + + if paths: + args.append("--") + args.extend(paths) + + kwargs["as_process"] = True + if create_patch: + self.repo.git(c="diff.mnemonicPrefix=false") + proc = self.repo.git.diff(*args, **kwargs) + + diff_method = ( + git_diff.Diff._index_from_patch_format if create_patch else git_diff.Diff._index_from_raw_format + ) + index = diff_method(self.repo, proc) + + proc.wait() + return index + + # Index against anything but None is a reverse diff with the respective item. + # Handle existing -R flags properly. + # Transform strings to the object so that we can call diff on it. if isinstance(other, str): other = self.repo.rev_parse(other) # END object conversion @@ -1441,12 +1629,18 @@ def diff( # Invert the existing R flag. cur_val = kwargs.get("R", False) kwargs["R"] = not cur_val - return other.diff(self.Index, paths, create_patch, **kwargs) + return other.diff( + self.INDEX, + paths, + create_patch, + allow_unsafe_options=allow_unsafe_options, + **kwargs, + ) # END diff against other item handling # If other is not None here, something is wrong. if other is not None: - raise ValueError("other must be None, Diffable.Index, a Tree or Commit, was %r" % other) + raise ValueError("other must be None, Diffable.INDEX, a Tree or Commit, was %r" % other) # Diff against working copy - can be handled by superclass natively. - return super().diff(other, paths, create_patch, **kwargs) + return super().diff(other, paths, create_patch, allow_unsafe_options=allow_unsafe_options, **kwargs) diff --git a/git/index/fun.py b/git/index/fun.py index 580493f6d..886e10de9 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -1,24 +1,32 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Standalone functions to accompany the index implementation and make it more versatile.""" +"""Standalone functions to accompany the index implementation and make it more +versatile.""" + +__all__ = [ + "write_cache", + "read_cache", + "write_tree_from_cache", + "entry_key", + "stat_mode_to_index_mode", + "S_IFGITLINK", + "run_commit_hook", + "hook_path", +] from io import BytesIO import os import os.path as osp from pathlib import Path -from stat import ( - S_IFDIR, - S_IFLNK, - S_ISLNK, - S_ISDIR, - S_IFMT, - S_IFREG, - S_IXUSR, -) +from stat import S_IFDIR, S_IFLNK, S_IFMT, S_IFREG, S_ISDIR, S_ISLNK, S_IXUSR import subprocess +import sys + +from gitdb.base import IStream +from gitdb.typ import str_tree_type -from git.cmd import handle_process_output, safer_popen +from git.cmd import Git, handle_process_output, safer_popen from git.compat import defenc, force_bytes, force_text, safe_decode from git.exc import HookExecutionError, UnmergedEntriesError from git.objects.fun import ( @@ -27,10 +35,8 @@ tree_to_stream, ) from git.util import IndexFileSHA1Writer, finalize_process -from gitdb.base import IStream -from gitdb.typ import str_tree_type -from .typ import BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT +from .typ import CE_EXTENDED, BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT from .util import pack, unpack # typing ----------------------------------------------------------------------------- @@ -40,63 +46,171 @@ from git.types import PathLike if TYPE_CHECKING: - from .base import IndexFile from git.db import GitCmdObjectDB from git.objects.tree import TreeCacheTup - # from git.objects.fun import EntryTupOrNone + from .base import IndexFile # ------------------------------------------------------------------------------------ - S_IFGITLINK = S_IFLNK | S_IFDIR """Flags for a submodule.""" CE_NAMEMASK_INV = ~CE_NAMEMASK -__all__ = ( - "write_cache", - "read_cache", - "write_tree_from_cache", - "entry_key", - "stat_mode_to_index_mode", - "S_IFGITLINK", - "run_commit_hook", - "hook_path", -) - def hook_path(name: str, git_dir: PathLike) -> str: """:return: path to the given named hook in the given git repository directory""" return osp.join(git_dir, "hooks", name) +def _commit_hook_path(name: str, index: "IndexFile") -> str: + """:return: path to the named commit hook, respecting Git's core.hooksPath.""" + with index.repo.config_reader() as config: + hooks_dir = config.get("core", "hooksPath", fallback="") + + if not hooks_dir: + return hook_path(name, index.repo.git_dir) + + return osp.abspath(osp.join(index.repo.working_dir, osp.expanduser(hooks_dir), name)) + + def _has_file_extension(path: str) -> str: return osp.splitext(path)[1] +def _is_in_windows_system_root(path: str) -> bool: + """Return whether ``path`` is inside the Windows installation directory.""" + system_root = os.environ.get("SystemRoot") + if not system_root: + return False + + system_root = osp.normcase(osp.realpath(system_root)) + path = osp.normcase(osp.realpath(path)) + try: + return osp.commonpath((system_root, path)) == system_root + except ValueError: + # Paths on different drives have no common path on Windows. + return False + + +def _which_from_path(command: str) -> Union[str, None]: + """Resolve ``command`` from PATH, excluding the Windows installation.""" + for directory in os.get_exec_path(): + # Unlike POSIX, Windows does not define an empty PATH entry as the current + # directory. Skip it rather than letting abspath() turn it into one. + if not directory: + continue + directory = osp.abspath(directory) + candidate = osp.join(directory, command) + # SystemRoot contains the WSL launcher stubs. They are valid executables but + # not suitable for running a Windows Git hook: the hook path and environment + # were prepared for Git for Windows, and WSL may have no distribution at all. + if _is_in_windows_system_root(candidate): + continue + if osp.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + return None + + +_GIT_FOR_WINDOWS_PREFIXES = ("mingw64", "mingw32", "clangarm64", "clang64", "clang32", "ucrt64") + + +def _git_for_windows_root() -> Union[str, None]: + """Infer a standard Git for Windows root from GitPython's selected executable.""" + git_executable = os.fspath(Git.GIT_PYTHON_GIT_EXECUTABLE or Git.git_exec_name) + if osp.dirname(git_executable): + # CreateProcess resolves a relative executable path containing a directory + # from the parent process cwd, even when Popen supplies a different child cwd. + git_executable = osp.abspath(git_executable) + else: + # GitPython deliberately retains a bare executable name so later PATH changes + # affect Git commands. Resolve it with the same PATH snapshot used for Bash. + names = (git_executable,) if _has_file_extension(git_executable) else (git_executable, f"{git_executable}.exe") + for name in names: + resolved = _which_from_path(name) + if resolved is not None: + git_executable = resolved + break + else: + git_executable = "" + if not git_executable: + return None + if osp.basename(git_executable).lower() not in ("git", "git.exe"): + return None + + executable_dir = osp.dirname(git_executable) + directory_name = osp.basename(executable_dir).lower() + if directory_name == "cmd": + # The normal system-wide PATH entry is /cmd. + return osp.dirname(executable_dir) + if directory_name == "bin": + prefix = osp.dirname(executable_dir) + if osp.basename(prefix).lower() in _GIT_FOR_WINDOWS_PREFIXES: + # Git Bash commonly exposes //bin/git.exe. + return osp.dirname(prefix) + if osp.basename(prefix).lower() != "usr": + # An explicitly configured Git may be the root-level bin/git.exe. Do + # not make the same inference from usr/bin: unlike the recognized + # platform prefixes, "usr" has no reliably bounded parent layout. + return prefix + return None + + +def _git_for_windows_bash() -> Union[str, None]: + """Return Bash from the Git for Windows installation selected by GitPython.""" + git_root = _git_for_windows_root() + if git_root is None: + return None + + # Match gix-path's precedence: prefer the lightweight bin shim, then the + # underlying usr/bin executable. Both belong to the same installation as Git. + for relative_path in ("bin/bash.exe", "usr/bin/bash.exe"): + candidate = osp.join(git_root, *relative_path.split("/")) + if osp.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + return None + + def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: """Run the commit hook of the given name. Silently ignore hooks that do not exist. - :param name: name of hook, like 'pre-commit' - :param index: IndexFile instance - :param args: Arguments passed to hook file - :raises HookExecutionError: + :param name: + Name of hook, like ``pre-commit``. + + :param index: + :class:`~git.index.base.IndexFile` instance. + + :param args: + Arguments passed to hook file. + + :raise git.exc.HookExecutionError: """ - hp = hook_path(name, index.repo.git_dir) + hp = _commit_hook_path(name, index) if not os.access(hp, os.X_OK): return env = os.environ.copy() - env["GIT_INDEX_FILE"] = safe_decode(str(index.path)) + env["GIT_INDEX_FILE"] = safe_decode(os.fspath(index.path)) env["GIT_EDITOR"] = ":" cmd = [hp] try: - if os.name == "nt" and not _has_file_extension(hp): + if sys.platform == "win32" and not _has_file_extension(hp): # Windows only uses extensions to determine how to open files # (doesn't understand shebangs). Try using bash to run the hook. - relative_hp = Path(hp).relative_to(index.repo.working_dir).as_posix() - cmd = ["bash.exe", relative_hp] + try: + bash_hp = osp.relpath(hp, index.repo.working_dir) + except ValueError: + # Different drives have no relative path on Windows. Git Bash accepts + # an absolute path in this form, although a relative path is preferable + # because it also works with the Windows Subsystem for Linux wrapper. + bash_hp = hp + # Prefer Bash associated with GitPython's selected Git installation. If + # that layout is not recognized, use an explicitly configured non-system + # PATH entry. Preserve the bare fallback for installations that previously + # relied on WSL or another CreateProcess-resolved Bash. + bash_executable = _git_for_windows_bash() or _which_from_path("bash.exe") or "bash.exe" + cmd = [bash_executable, Path(bash_hp).as_posix()] process = safer_popen( cmd + list(args), @@ -121,8 +235,8 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: def stat_mode_to_index_mode(mode: int) -> int: - """Convert the given mode from a stat call to the corresponding index mode - and return it.""" + """Convert the given mode from a stat call to the corresponding index mode and + return it.""" if S_ISLNK(mode): # symlinks return S_IFLNK if S_ISDIR(mode) or S_IFMT(mode) == S_IFGITLINK: # submodules @@ -138,16 +252,19 @@ def write_cache( ) -> None: """Write the cache represented by entries to a stream. - :param entries: **sorted** list of entries + :param entries: + **Sorted** list of entries. - :param stream: stream to wrap into the AdapterStreamCls - it is used for - final output. + :param stream: + Stream to wrap into the AdapterStreamCls - it is used for final output. - :param ShaStreamCls: Type to use when writing to the stream. It produces a sha - while writing to it, before the data is passed on to the wrapped stream + :param ShaStreamCls: + Type to use when writing to the stream. It produces a sha while writing to it, + before the data is passed on to the wrapped stream. - :param extension_data: any kind of data to write as a trailer, it must begin - a 4 byte identifier, followed by its size (4 bytes). + :param extension_data: + Any kind of data to write as a trailer, it must begin a 4 byte identifier, + followed by its size (4 bytes). """ # Wrap the stream into a compatible writer. stream_sha = ShaStreamCls(stream) @@ -156,7 +273,7 @@ def write_cache( write = stream_sha.write # Header - version = 2 + version = 3 if any(entry.extended_flags for entry in entries) else 2 write(b"DIRC") write(pack(">LL", version, len(entries))) @@ -170,6 +287,8 @@ def write_cache( plen = len(path) & CE_NAMEMASK # Path length assert plen == len(path), "Path %s too long to fit into index" % entry.path flags = plen | (entry.flags & CE_NAMEMASK_INV) # Clear possible previous values. + if entry.extended_flags: + flags |= CE_EXTENDED write( pack( ">LLLLLL20sH", @@ -183,6 +302,8 @@ def write_cache( flags, ) ) + if entry.extended_flags: + write(pack(">H", entry.extended_flags)) write(path) real_size = (tell() - beginoffset + 8) & ~7 write(b"\0" * ((beginoffset + real_size) - tell())) @@ -197,22 +318,25 @@ def write_cache( def read_header(stream: IO[bytes]) -> Tuple[int, int]: - """Return tuple(version_long, num_entries) from the given stream""" + """Return tuple(version_long, num_entries) from the given stream.""" type_id = stream.read(4) if type_id != b"DIRC": raise AssertionError("Invalid index file header: %r" % type_id) unpacked = cast(Tuple[int, int], unpack(">LL", stream.read(4 * 2))) version, num_entries = unpacked - # TODO: Handle version 3: extended data, see read-cache.c. - assert version in (1, 2) + assert version in (1, 2, 3), "Unsupported git index version %i, only 1, 2, and 3 are supported" % version return version, num_entries def entry_key(*entry: Union[BaseIndexEntry, PathLike, int]) -> Tuple[PathLike, int]: - """:return: Key suitable to be used for the index.entries dictionary + """ + :return: + Key suitable to be used for the + :attr:`index.entries ` dictionary. - :param entry: One instance of type BaseIndexEntry or the path and the stage + :param entry: + One instance of type BaseIndexEntry or the path and the stage. """ # def is_entry_key_tup(entry_key: Tuple) -> TypeGuard[Tuple[PathLike, int]]: @@ -234,12 +358,14 @@ def read_cache( ) -> Tuple[int, Dict[Tuple[PathLike, int], "IndexEntry"], bytes, bytes]: """Read a cache file from the given stream. - :return: tuple(version, entries_dict, extension_data, content_sha) + :return: + tuple(version, entries_dict, extension_data, content_sha) - * version is the integer version number. - * entries dict is a dictionary which maps IndexEntry instances to a path at a stage. - * extension_data is '' or 4 bytes of type + 4 bytes of size + size bytes. - * content_sha is a 20 byte sha on all cache file contents. + * *version* is the integer version number. + * *entries_dict* is a dictionary which maps IndexEntry instances to a path at a + stage. + * *extension_data* is ``""`` or 4 bytes of type + 4 bytes of size + size bytes. + * *content_sha* is a 20 byte sha on all cache file contents. """ version, num_entries = read_header(stream) count = 0 @@ -252,12 +378,15 @@ def read_cache( ctime = unpack(">8s", read(8))[0] mtime = unpack(">8s", read(8))[0] (dev, ino, mode, uid, gid, size, sha, flags) = unpack(">LLLLLL20sH", read(20 + 4 * 6 + 2)) + extended_flags = 0 + if flags & CE_EXTENDED: + extended_flags = unpack(">H", read(2))[0] path_size = flags & CE_NAMEMASK path = read(path_size).decode(defenc) real_size = (tell() - beginoffset + 8) & ~7 read((beginoffset + real_size) - tell()) - entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size)) + entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size, extended_flags)) # entry_key would be the method to use, but we save the effort. entries[(path, entry.stage)] = entry count += 1 @@ -270,9 +399,9 @@ def read_cache( # 4 bytes length of chunk # Repeated 0 - N times extension_data = stream.read(~0) - assert ( - len(extension_data) > 19 - ), "Index Footer was not at least a sha on content as it was only %i bytes in size" % len(extension_data) + assert len(extension_data) > 19, ( + "Index Footer was not at least a sha on content as it was only %i bytes in size" % len(extension_data) + ) content_sha = extension_data[-20:] @@ -285,15 +414,25 @@ def read_cache( def write_tree_from_cache( entries: List[IndexEntry], odb: "GitCmdObjectDB", sl: slice, si: int = 0 ) -> Tuple[bytes, List["TreeCacheTup"]]: - """Create a tree from the given sorted list of entries and put the respective + R"""Create a tree from the given sorted list of entries and put the respective trees into the given object database. - :param entries: **Sorted** list of IndexEntries - :param odb: Object database to store the trees in - :param si: Start index at which we should start creating subtrees - :param sl: Slice indicating the range we should process on the entries list - :return: tuple(binsha, list(tree_entry, ...)) a tuple of a sha and a list of - tree entries being a tuple of hexsha, mode, name + :param entries: + **Sorted** list of :class:`~git.index.typ.IndexEntry`\s. + + :param odb: + Object database to store the trees in. + + :param si: + Start index at which we should start creating subtrees. + + :param sl: + Slice indicating the range we should process on the entries list. + + :return: + tuple(binsha, list(tree_entry, ...)) + + A tuple of a sha and a list of tree entries being a tuple of hexsha, mode, name. """ tree_items: List["TreeCacheTup"] = [] @@ -346,15 +485,17 @@ def _tree_entry_to_baseindexentry(tree_entry: "TreeCacheTup", stage: int) -> Bas def aggressive_tree_merge(odb: "GitCmdObjectDB", tree_shas: Sequence[bytes]) -> List[BaseIndexEntry]: - """ - :return: List of BaseIndexEntries representing the aggressive merge of the given - trees. All valid entries are on stage 0, whereas the conflicting ones are left - on stage 1, 2 or 3, whereas stage 1 corresponds to the common ancestor tree, - 2 to our tree and 3 to 'their' tree. - - :param tree_shas: 1, 2 or 3 trees as identified by their binary 20 byte shas. - If 1 or two, the entries will effectively correspond to the last given tree. - If 3 are given, a 3 way merge is performed. + R""" + :return: + List of :class:`~git.index.typ.BaseIndexEntry`\s representing the aggressive + merge of the given trees. All valid entries are on stage 0, whereas the + conflicting ones are left on stage 1, 2 or 3, whereas stage 1 corresponds to the + common ancestor tree, 2 to our tree and 3 to 'their' tree. + + :param tree_shas: + 1, 2 or 3 trees as identified by their binary 20 byte shas. If 1 or two, the + entries will effectively correspond to the last given tree. If 3 are given, a 3 + way merge is performed. """ out: List[BaseIndexEntry] = [] diff --git a/git/index/typ.py b/git/index/typ.py index 894e6f16d..927633a9f 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -3,16 +3,18 @@ """Additional types used by the index.""" +__all__ = ["BlobFilter", "BaseIndexEntry", "IndexEntry", "StageType"] + from binascii import b2a_hex from pathlib import Path -from .util import pack, unpack from git.objects import Blob +from .util import pack, unpack # typing ---------------------------------------------------------------------- -from typing import NamedTuple, Sequence, TYPE_CHECKING, Tuple, Union, cast, List +from typing import NamedTuple, Sequence, TYPE_CHECKING, Tuple, Union, cast from git.types import PathLike @@ -23,8 +25,6 @@ # --------------------------------------------------------------------------------- -__all__ = ("BlobFilter", "BaseIndexEntry", "IndexEntry", "StageType") - # { Invariants CE_NAMEMASK = 0x0FFF CE_STAGEMASK = 0x3000 @@ -32,13 +32,16 @@ CE_VALID = 0x8000 CE_STAGESHIFT = 12 +CE_EXT_SKIP_WORKTREE = 0x4000 +CE_EXT_INTENT_TO_ADD = 0x2000 + # } END invariants class BlobFilter: - """ - Predicate to be used by iter_blobs allowing to filter only return blobs which - match the given list of directories or files. + """Predicate to be used by + :meth:`IndexFile.iter_blobs ` allowing to + filter only return blobs which match the given list of directories or files. The given paths are given relative to the repository. """ @@ -48,8 +51,8 @@ class BlobFilter: def __init__(self, paths: Sequence[PathLike]) -> None: """ :param paths: - Tuple or list of paths which are either pointing to directories or - to files relative to the current repository + Tuple or list of paths which are either pointing to directories or to files + relative to the current repository. """ self.paths = paths @@ -58,9 +61,10 @@ def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool: blob_path: Path = blob_pathlike if isinstance(blob_pathlike, Path) else Path(blob_pathlike) for pathlike in self.paths: path: Path = pathlike if isinstance(pathlike, Path) else Path(pathlike) - # TODO: Change to use `PosixPath.is_relative_to` once Python 3.8 is no longer supported. - filter_parts: List[str] = path.parts - blob_parts: List[str] = blob_path.parts + # TODO: Change to use `PosixPath.is_relative_to` once Python 3.8 is no + # longer supported. + filter_parts = path.parts + blob_parts = blob_path.parts if len(filter_parts) > len(blob_parts): continue if all(i == j for i, j in zip(filter_parts, blob_parts)): @@ -69,8 +73,11 @@ def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool: class BaseIndexEntryHelper(NamedTuple): - """Typed namedtuple to provide named attribute access for BaseIndexEntry. - Needed to allow overriding __new__ in child class to preserve backwards compat.""" + """Typed named tuple to provide named attribute access for :class:`BaseIndexEntry`. + + This is needed to allow overriding ``__new__`` in child class to preserve backwards + compatibility. + """ mode: int binsha: bytes @@ -83,25 +90,28 @@ class BaseIndexEntryHelper(NamedTuple): uid: int = 0 gid: int = 0 size: int = 0 + # version 3 extended flags, only when (flags & CE_EXTENDED) is set + extended_flags: int = 0 class BaseIndexEntry(BaseIndexEntryHelper): - """Small brother of an index entry which can be created to describe changes + R"""Small brother of an index entry which can be created to describe changes done to the index in which case plenty of additional information is not required. - As the first 4 data members match exactly to the IndexEntry type, methods - expecting a BaseIndexEntry can also handle full IndexEntries even if they - use numeric indices for performance reasons. + As the first 4 data members match exactly to the :class:`IndexEntry` type, methods + expecting a :class:`BaseIndexEntry` can also handle full :class:`IndexEntry`\s even + if they use numeric indices for performance reasons. """ def __new__( cls, inp_tuple: Union[ Tuple[int, bytes, int, PathLike], - Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int], + Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int, int], ], ) -> "BaseIndexEntry": - """Override __new__ to allow construction from a tuple for backwards compatibility""" + """Override ``__new__`` to allow construction from a tuple for backwards + compatibility.""" return super().__new__(cls, *inp_tuple) def __str__(self) -> str: @@ -122,12 +132,21 @@ def stage(self) -> int: * 0 = default stage * 1 = stage before a merge or common ancestor entry in case of a 3 way merge * 2 = stage of entries from the 'left' side of the merge - * 3 = stage of entries from the right side of the merge + * 3 = stage of entries from the 'right' side of the merge - :note: For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html + :note: + For more information, see :manpage:`git-read-tree(1)`. """ return (self.flags & CE_STAGEMASK) >> CE_STAGESHIFT + @property + def skip_worktree(self) -> bool: + return (self.extended_flags & CE_EXT_SKIP_WORKTREE) > 0 + + @property + def intent_to_add(self) -> bool: + return (self.extended_flags & CE_EXT_INTENT_TO_ADD) > 0 + @classmethod def from_blob(cls, blob: Blob, stage: int = 0) -> "BaseIndexEntry": """:return: Fully equipped BaseIndexEntry at the given stage""" @@ -139,7 +158,8 @@ def to_blob(self, repo: "Repo") -> Blob: class IndexEntry(BaseIndexEntry): - """Allows convenient access to IndexEntry data without completely unpacking it. + """Allows convenient access to index entry data as defined in + :class:`BaseIndexEntry` without completely unpacking it. Attributes usually accessed often are cached in the tuple whereas others are unpacked on demand. @@ -158,20 +178,21 @@ def ctime(self) -> Tuple[int, int]: @property def mtime(self) -> Tuple[int, int]: - """See ctime property, but returns modification time.""" + """See :attr:`ctime` property, but returns modification time.""" return cast(Tuple[int, int], unpack(">LL", self.mtime_bytes)) @classmethod def from_base(cls, base: "BaseIndexEntry") -> "IndexEntry": """ :return: - Minimal entry as created from the given BaseIndexEntry instance. + Minimal entry as created from the given :class:`BaseIndexEntry` instance. Missing values will be set to null-like values. - :param base: Instance of type :class:`BaseIndexEntry` + :param base: + Instance of type :class:`BaseIndexEntry`. """ time = pack(">LL", 0, 0) - return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0)) + return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0)) # type: ignore[arg-type] @classmethod def from_blob(cls, blob: Blob, stage: int = 0) -> "IndexEntry": @@ -190,5 +211,5 @@ def from_blob(cls, blob: Blob, stage: int = 0) -> "IndexEntry": 0, 0, blob.size, - ) + ) # type: ignore[arg-type] ) diff --git a/git/index/util.py b/git/index/util.py index 125925783..982a5afb7 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -3,6 +3,8 @@ """Index utilities.""" +__all__ = ["TemporaryFileSwap", "post_clear_cache", "default_index", "git_working_dir"] + import contextlib from functools import wraps import os @@ -13,7 +15,7 @@ # typing ---------------------------------------------------------------------- -from typing import Any, Callable, TYPE_CHECKING, Optional, Type +from typing import Any, Callable, TYPE_CHECKING, Optional, Type, cast from git.types import Literal, PathLike, _T @@ -22,20 +24,15 @@ # --------------------------------------------------------------------------------- - -__all__ = ("TemporaryFileSwap", "post_clear_cache", "default_index", "git_working_dir") - # { Aliases pack = struct.pack unpack = struct.unpack - - # } END aliases class TemporaryFileSwap: - """Utility class moving a file to a temporary location within the same directory - and moving it back on to where on object deletion.""" + """Utility class moving a file to a temporary location within the same directory and + moving it back on to where on object deletion.""" __slots__ = ("file_path", "tmp_file_path") @@ -65,13 +62,10 @@ def __exit__( def post_clear_cache(func: Callable[..., _T]) -> Callable[..., _T]: - """Decorator for functions that alter the index using the git command. This would - invalidate our possibly existing entries dictionary which is why it must be - deleted to allow it to be lazily reread later. + """Decorator for functions that alter the index using the git command. - :note: - This decorator will not be required once all functions are implemented - natively which in fact is possible, but probably not feasible performance wise. + When a git command alters the index, this invalidates our possibly existing entries + dictionary, which is why it must be deleted to allow it to be lazily reread later. """ @wraps(func) @@ -112,7 +106,7 @@ def git_working_dir(func: Callable[..., _T]) -> Callable[..., _T]: @wraps(func) def set_git_working_dir(self: "IndexFile", *args: Any, **kwargs: Any) -> _T: cur_wd = os.getcwd() - os.chdir(str(self.repo.working_tree_dir)) + os.chdir(cast(PathLike, self.repo.working_tree_dir)) try: return func(self, *args, **kwargs) finally: diff --git a/git/objects/__init__.py b/git/objects/__init__.py index fc8d8fbb2..4447ca50d 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -3,22 +3,23 @@ """Import all submodules' main classes into the package space.""" -import inspect +__all__ = [ + "IndexObject", + "Object", + "Blob", + "Commit", + "Submodule", + "UpdateProgress", + "RootModule", + "RootUpdateProgress", + "TagObject", + "Tree", + "TreeModifier", +] -from .base import * # noqa: F403 -from .blob import * # noqa: F403 -from .commit import * # noqa: F403 -from .submodule import util as smutil -from .submodule.base import * # noqa: F403 -from .submodule.root import * # noqa: F403 -from .tag import * # noqa: F403 -from .tree import * # noqa: F403 - -# Fix import dependency - add IndexObject to the util module, so that it can be -# imported by the submodule.base. -smutil.IndexObject = IndexObject # type: ignore[attr-defined] # noqa: F405 -smutil.Object = Object # type: ignore[attr-defined] # noqa: F405 -del smutil - -# Must come after submodule was made available. -__all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] +from .base import IndexObject, Object +from .blob import Blob +from .commit import Commit +from .submodule import RootModule, RootUpdateProgress, Submodule, UpdateProgress +from .tag import TagObject +from .tree import Tree, TreeModifier diff --git a/git/objects/base.py b/git/objects/base.py index 934fb40bc..faf600c6b 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -3,41 +3,71 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from git.exc import WorkTreeRepositoryUnsupported -from git.util import LazyMixin, join_path_native, stream_copy, bin_to_hex +__all__ = ["Object", "IndexObject"] -import gitdb.typ as dbtyp import os.path as osp -from .util import get_object_type_by_name +import gitdb.typ as dbtyp + +from git.exc import WorkTreeRepositoryUnsupported +from git.util import LazyMixin, bin_to_hex, join_path_native, stream_copy +from .util import get_object_type_by_name # typing ------------------------------------------------------------------ from typing import Any, TYPE_CHECKING, Union -from git.types import PathLike, Commit_ish, Lit_commit_ish +from git.types import AnyGitObject, GitObjectTypeString, PathLike if TYPE_CHECKING: - from git.repo import Repo from gitdb.base import OStream - from .tree import Tree + + from git.refs.reference import Reference + from git.repo import Repo + from .blob import Blob from .submodule.base import Submodule - from git.refs.reference import Reference + from .tree import Tree IndexObjUnion = Union["Tree", "Blob", "Submodule"] # -------------------------------------------------------------------------- -_assertion_msg_format = "Created object %r whose python type %r disagrees with the actual git object type %r" +class Object(LazyMixin): + """Base class for classes representing git object types. -__all__ = ("Object", "IndexObject") + The following four leaf classes represent specific kinds of git objects: + * :class:`Blob ` + * :class:`Tree ` + * :class:`Commit ` + * :class:`TagObject ` -class Object(LazyMixin): - """An Object which may be Blobs, Trees, Commits and Tags.""" + See :manpage:`gitglossary(7)` on: + + * "object": https://git-scm.com/docs/gitglossary#def_object + * "object type": https://git-scm.com/docs/gitglossary#def_object_type + * "blob": https://git-scm.com/docs/gitglossary#def_blob_object + * "tree object": https://git-scm.com/docs/gitglossary#def_tree_object + * "commit object": https://git-scm.com/docs/gitglossary#def_commit_object + * "tag object": https://git-scm.com/docs/gitglossary#def_tag_object + + :note: + See the :class:`~git.types.AnyGitObject` union type of the four leaf subclasses + that represent actual git object types. + + :note: + :class:`~git.objects.submodule.base.Submodule` is defined under the hierarchy + rooted at this :class:`Object` class, even though submodules are not really a + type of git object. (This also applies to its + :class:`~git.objects.submodule.root.RootModule` subclass.) + + :note: + This :class:`Object` class should not be confused with :class:`object` (the root + of the class hierarchy in Python). + """ NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = b"\0" * 20 @@ -51,15 +81,32 @@ class Object(LazyMixin): __slots__ = ("repo", "binsha", "size") - type: Union[Lit_commit_ish, None] = None + type: Union[GitObjectTypeString, None] = None + """String identifying (a concrete :class:`Object` subtype for) a git object type. + + The subtypes that this may name correspond to the kinds of git objects that exist, + i.e., the objects that may be present in a git repository. + + :note: + Most subclasses represent specific types of git objects and override this class + attribute accordingly. This attribute is ``None`` in the :class:`Object` base + class, as well as the :class:`IndexObject` intermediate subclass, but never + ``None`` in concrete leaf subclasses representing specific git object types. + + :note: + See also :class:`~git.types.GitObjectTypeString`. + """ - def __init__(self, repo: "Repo", binsha: bytes): + def __init__(self, repo: "Repo", binsha: bytes) -> None: """Initialize an object by identifying it by its binary sha. - All keyword arguments will be set on demand if None. - :param repo: repository this object is located in + All keyword arguments will be set on demand if ``None``. - :param binsha: 20 byte SHA1 + :param repo: + Repository this object is located in. + + :param binsha: + 20 byte SHA1 """ super().__init__() self.repo = repo @@ -70,26 +117,30 @@ def __init__(self, repo: "Repo", binsha: bytes): ) @classmethod - def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Commit_ish: + def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> AnyGitObject: """ - :return: New :class:`Object`` instance of a type appropriate to the object type - behind `id`. The id of the newly created object will be a binsha even though - the input id may have been a Reference or Rev-Spec. + :return: + New :class:`Object` instance of a type appropriate to the object type behind + `id`. The id of the newly created object will be a binsha even though the + input id may have been a :class:`~git.refs.reference.Reference` or rev-spec. - :param id: reference, rev-spec, or hexsha + :param id: + :class:`~git.refs.reference.Reference`, rev-spec, or hexsha. - :note: This cannot be a ``__new__`` method as it would always call - :meth:`__init__` with the input id which is not necessarily a binsha. + :note: + This cannot be a ``__new__`` method as it would always call :meth:`__init__` + with the input id which is not necessarily a binsha. """ return repo.rev_parse(str(id)) @classmethod - def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Commit_ish: + def new_from_sha(cls, repo: "Repo", sha1: bytes) -> AnyGitObject: """ - :return: new object instance of a type appropriate to represent the given - binary sha1 + :return: + New object instance of a type appropriate to represent the given binary sha1 - :param sha1: 20 byte binary sha1 + :param sha1: + 20 byte binary sha1. """ if sha1 == cls.NULL_BIN_SHA: # The NULL binsha is always the root commit. @@ -104,19 +155,18 @@ def _set_cache_(self, attr: str) -> None: """Retrieve object information.""" if attr == "size": oinfo = self.repo.odb.info(self.binsha) - self.size = oinfo.size # type: int - # assert oinfo.type == self.type, _assertion_msg_format % (self.binsha, oinfo.type, self.type) + self.size = oinfo.size # type: int else: super()._set_cache_(attr) def __eq__(self, other: Any) -> bool: - """:return: True if the objects have the same SHA1""" + """:return: ``True`` if the objects have the same SHA1""" if not hasattr(other, "binsha"): return False return self.binsha == other.binsha def __ne__(self, other: Any) -> bool: - """:return: True if the objects do not have the same SHA1""" + """:return: ``True`` if the objects do not have the same SHA1""" if not hasattr(other, "binsha"): return True return self.binsha != other.binsha @@ -126,11 +176,11 @@ def __hash__(self) -> int: return hash(self.binsha) def __str__(self) -> str: - """:return: string of our SHA1 as understood by all git commands""" + """:return: String of our SHA1 as understood by all git commands""" return self.hexsha def __repr__(self) -> str: - """:return: string with pythonic representation of our object""" + """:return: String with pythonic representation of our object""" return '' % (self.__class__.__name__, self.hexsha) @property @@ -142,17 +192,22 @@ def hexsha(self) -> str: @property def data_stream(self) -> "OStream": """ - :return: File Object compatible stream to the uncompressed raw data of the object + :return: + File-object compatible stream to the uncompressed raw data of the object - :note: Returned streams must be read in order. + :note: + Returned streams must be read in order. """ return self.repo.odb.stream(self.binsha) def stream_data(self, ostream: "OStream") -> "Object": """Write our data directly to the given output stream. - :param ostream: File object compatible stream object. - :return: self + :param ostream: + File-object compatible stream object. + + :return: + self """ istream = self.repo.odb.stream(self.binsha) stream_copy(istream, ostream) @@ -160,8 +215,13 @@ def stream_data(self, ostream: "OStream") -> "Object": class IndexObject(Object): - """Base for all objects that can be part of the index file, namely Tree, Blob and - SubModule objects.""" + """Base for all objects that can be part of the index file. + + The classes representing git object types that can be part of the index file are + :class:`~git.objects.tree.Tree` and :class:`~git.objects.blob.Blob`. In addition, + :class:`~git.objects.submodule.base.Submodule`, which is not really a git object + type but can be part of an index file, is also a subclass. + """ __slots__ = ("path", "mode") @@ -175,16 +235,22 @@ def __init__( mode: Union[None, int] = None, path: Union[None, PathLike] = None, ) -> None: - """Initialize a newly instanced IndexObject. + """Initialize a newly instanced :class:`IndexObject`. + + :param repo: + The :class:`~git.repo.base.Repo` we are located in. + + :param binsha: + 20 byte sha1. - :param repo: The :class:`~git.repo.base.Repo` we are located in. - :param binsha: 20 byte sha1. :param mode: - The stat compatible file mode as int, use the :mod:`stat` module to evaluate - the information. + The stat-compatible file mode as :class:`int`. + Use the :mod:`stat` module to evaluate the information. + :param path: The path to the file in the file system, relative to the git repository root, like ``file.ext`` or ``folder/other.ext``. + :note: Path may not be set if the index object has been created directly, as it cannot be retrieved without knowing the parent tree. @@ -198,8 +264,8 @@ def __init__( def __hash__(self) -> int: """ :return: - Hash of our path as index items are uniquely identifiable by path, not - by their data! + Hash of our path as index items are uniquely identifiable by path, not by + their data! """ return hash(self.path) @@ -226,7 +292,8 @@ def abspath(self) -> PathLike: Absolute path to this index object in the file system (as opposed to the :attr:`path` field which is a path relative to the git repository). - The returned path will be native to the system and contains '\' on Windows. + The returned path will be native to the system and contains ``\`` on + Windows. """ if self.repo.working_tree_dir is not None: return join_path_native(self.repo.working_tree_dir, self.path) diff --git a/git/objects/blob.py b/git/objects/blob.py index 6d7e859af..f7d49c9cc 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -3,16 +3,26 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["Blob"] + from mimetypes import guess_type -from . import base +import os +import sys -from git.types import Literal +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal -__all__ = ("Blob",) +from . import base class Blob(base.IndexObject): - """A Blob encapsulates a git blob object.""" + """A Blob encapsulates a git blob object. + + See :manpage:`gitglossary(7)` on "blob": + https://git-scm.com/docs/gitglossary#def_blob_object + """ DEFAULT_MIME_TYPE = "text/plain" type: Literal["blob"] = "blob" @@ -27,11 +37,13 @@ class Blob(base.IndexObject): @property def mime_type(self) -> str: """ - :return: String describing the mime type of this file (based on the filename) + :return: + String describing the mime type of this file (based on the filename) - :note: Defaults to 'text/plain' in case the actual file type is unknown. + :note: + Defaults to ``text/plain`` in case the actual file type is unknown. """ guesses = None if self.path: - guesses = guess_type(str(self.path)) + guesses = guess_type(os.fspath(self.path)) return guesses and guesses[0] or self.DEFAULT_MIME_TYPE diff --git a/git/objects/commit.py b/git/objects/commit.py index 5a069848c..45843eac2 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -3,67 +3,77 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["Commit"] + +from collections import defaultdict import datetime +from io import BytesIO +import logging +import os import re from subprocess import Popen, PIPE +import sys +from time import altzone, daylight, localtime, time, timezone +import warnings + from gitdb import IStream -from git.util import hex_to_bin, Actor, Stats, finalize_process -from git.diff import Diffable + from git.cmd import Git +from git.diff import Diffable +from git.util import Actor, Stats, finalize_process, hex_to_bin -from .tree import Tree from . import base +from .tree import Tree from .util import ( Serializable, TraversableIterableObj, - parse_date, altz_to_utctz_str, - parse_actor_and_date, from_timestamp, + parse_actor_and_date, + parse_date, ) -from time import time, daylight, altzone, timezone, localtime -import os -from io import BytesIO -import logging -from collections import defaultdict - - # typing ------------------------------------------------------------------ from typing import ( Any, + Dict, IO, Iterator, List, Sequence, Tuple, - Union, TYPE_CHECKING, + Union, cast, - Dict, ) -from git.types import PathLike, Literal +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +from git.types import PathLike if TYPE_CHECKING: - from git.repo import Repo from git.refs import SymbolicReference + from git.repo import Repo # ------------------------------------------------------------------------ -log = logging.getLogger("git.objects.commit") -log.addHandler(logging.NullHandler()) - -__all__ = ("Commit",) +_logger = logging.getLogger(__name__) class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): + """Wraps a git commit object. - """Wraps a git Commit object. + See :manpage:`gitglossary(7)` on "commit object": + https://git-scm.com/docs/gitglossary#def_commit_object - This class will act lazily on some of its attributes and will query the - value on demand only if it involves calling the git binary.""" + :note: + This class will act lazily on some of its attributes and will query the value on + demand only if it involves calling the git binary. + """ # ENVIRONMENT VARIABLES # Read when creating new commits. @@ -76,8 +86,14 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): # INVARIANTS default_encoding = "UTF-8" - # object configuration + # Options to :manpage:`git-rev-list(1)` that can overwrite files. + unsafe_git_rev_options = [ + "--output", + "-o", + ] + type: Literal["commit"] = "commit" + __slots__ = ( "tree", "author", @@ -91,8 +107,11 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): "encoding", "gpgsig", ) + _id_attribute_ = "hexsha" + parents: Sequence["Commit"] + def __init__( self, repo: "Repo", @@ -109,41 +128,52 @@ def __init__( encoding: Union[str, None] = None, gpgsig: Union[str, None] = None, ) -> None: - """Instantiate a new Commit. All keyword arguments taking None as default will - be implicitly set on first query. - - :param binsha: 20 byte sha1 - :param parents: tuple(Commit, ...) - A tuple of commit ids or actual Commits - :param tree: Tree object - :param author: Actor - The author Actor object + """Instantiate a new :class:`Commit`. All keyword arguments taking ``None`` as + default will be implicitly set on first query. + + :param binsha: + 20 byte sha1. + + :param tree: + A :class:`~git.objects.tree.Tree` object. + + :param author: + The author :class:`~git.util.Actor` object. + :param authored_date: int_seconds_since_epoch - The authored DateTime - use time.gmtime() to convert it into a - different format + The authored DateTime - use :func:`time.gmtime` to convert it into a + different format. + :param author_tz_offset: int_seconds_west_of_utc - The timezone that the authored_date is in - :param committer: Actor - The committer string + The timezone that the `authored_date` is in. + + :param committer: + The committer string, as an :class:`~git.util.Actor` object. + :param committed_date: int_seconds_since_epoch - The committed DateTime - use time.gmtime() to convert it into a - different format + The committed DateTime - use :func:`time.gmtime` to convert it into a + different format. + :param committer_tz_offset: int_seconds_west_of_utc - The timezone that the committed_date is in + The timezone that the `committed_date` is in. + :param message: string - The commit message + The commit message. + :param encoding: string - Encoding of the message, defaults to UTF-8 + Encoding of the message, defaults to UTF-8. + :param parents: - List or tuple of Commit objects which are our parent(s) in the commit - dependency graph + List or tuple of :class:`Commit` objects which are our parent(s) in the + commit dependency graph. - :return: git.Commit + :return: + :class:`Commit` :note: - Timezone information is in the same format and in the same sign - as what time.altzone returns. The sign is inverted compared to git's - UTC timezone. + Timezone information is in the same format and in the same sign as what + :func:`time.altzone` returns. The sign is inverted compared to git's UTC + timezone. """ super().__init__(repo, binsha) self.binsha = binsha @@ -180,8 +210,11 @@ def _get_intermediate_items(cls, commit: "Commit") -> Tuple["Commit", ...]: def _calculate_sha_(cls, repo: "Repo", commit: "Commit") -> bytes: """Calculate the sha of a commit. - :param repo: Repo object the commit should be part of - :param commit: Commit object for which to generate the sha + :param repo: + :class:`~git.repo.base.Repo` object the commit should be part of. + + :param commit: + :class:`Commit` object for which to generate the sha. """ stream = BytesIO() @@ -193,10 +226,10 @@ def _calculate_sha_(cls, repo: "Repo", commit: "Commit") -> bytes: return istream.binsha def replace(self, **kwargs: Any) -> "Commit": - """Create new commit object from existing commit object. + """Create new commit object from an existing commit object. - Any values provided as keyword arguments will replace the - corresponding attribute in the new object. + Any values provided as keyword arguments will replace the corresponding + attribute in the new object. """ attrs = {k: getattr(self, k) for k in self.__slots__} @@ -236,21 +269,36 @@ def summary(self) -> Union[str, bytes]: else: return self.message.split(b"\n", 1)[0] - def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> int: + def count( + self, + paths: Union[PathLike, Sequence[PathLike]] = "", + allow_unsafe_options: bool = False, + **kwargs: Any, + ) -> int: """Count the number of commits reachable from this commit. :param paths: - An optional path or a list of paths restricting the return value - to commits actually containing the paths. + An optional path or a list of paths restricting the return value to commits + actually containing the paths. + + :param allow_unsafe_options: + Allow unsafe options, like ``--output``. :param kwargs: - Additional options to be passed to git-rev-list. They must not alter - the output style of the command, or parsing will yield incorrect results. + Additional options to be passed to :manpage:`git-rev-list(1)`. They must not + alter the output style of the command, or parsing will yield incorrect + results. - :return: An int defining the number of reachable commits + :return: + An int defining the number of reachable commits """ - # Yes, it makes a difference whether empty paths are given or not in our case - # as the empty paths version will ignore merge commits for some reason. + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), unsafe_options=self.unsafe_git_rev_options + ) + + # Yes, it makes a difference whether empty paths are given or not in our case as + # the empty paths version will ignore merge commits for some reason. if paths: return len(self.repo.git.rev_list(self.hexsha, "--", paths, **kwargs).splitlines()) return len(self.repo.git.rev_list(self.hexsha, **kwargs).splitlines()) @@ -259,8 +307,11 @@ def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) def name_rev(self) -> str: """ :return: - String describing the commits hex sha based on the closest Reference. - Mostly useful for UI purposes + String describing the commits hex sha based on the closest + :class:`~git.refs.reference.Reference`. + + :note: + Mostly useful for UI purposes. """ return self.repo.git.name_rev(self) @@ -268,33 +319,42 @@ def name_rev(self) -> str: def iter_items( cls, repo: "Repo", - rev: Union[str, "Commit", "SymbolicReference"], # type: ignore + rev: Union[str, "Commit", "SymbolicReference"], paths: Union[PathLike, Sequence[PathLike]] = "", + allow_unsafe_options: bool = False, **kwargs: Any, ) -> Iterator["Commit"]: - """Find all commits matching the given criteria. + R"""Find all commits matching the given criteria. - :param repo: The Repo + :param repo: + The :class:`~git.repo.base.Repo`. - :param rev: Revision specifier, see git-rev-parse for viable options. + :param rev: + Revision specifier. See :manpage:`git-rev-parse(1)` for viable options. :param paths: - An optional path or list of paths, if set only Commits that include the path - or paths will be considered. + An optional path or list of paths. If set only :class:`Commit`\s that + include the path or paths will be considered. :param kwargs: - Optional keyword arguments to ``git rev-list`` where: + Optional keyword arguments to :manpage:`git-rev-list(1)` where: - * ``max_count`` is the maximum number of commits to fetch - * ``skip`` is the number of commits to skip - * ``since`` all commits since e.g. '1970-01-01' + * ``max_count`` is the maximum number of commits to fetch. + * ``skip`` is the number of commits to skip. + * ``since`` selects all commits since some date, e.g. ``"1970-01-01"``. - :return: Iterator yielding :class:`Commit` items. + :return: + Iterator yielding :class:`Commit` items. """ if "pretty" in kwargs: raise ValueError("--pretty cannot be used as parsing expects single sha's only") # END handle pretty + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([rev], kwargs), unsafe_options=cls.unsafe_git_rev_options + ) + # Use -- in all cases, to prevent possibility of ambiguous arguments. # See https://github.com/gitpython-developers/GitPython/issues/264. @@ -314,13 +374,17 @@ def iter_items( return cls._iter_from_process_or_stream(repo, proc) def iter_parents(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> Iterator["Commit"]: - """Iterate _all_ parents of this commit. + R"""Iterate *all* parents of this commit. :param paths: - Optional path or list of paths limiting the Commits to those that - contain at least one of the paths - :param kwargs: All arguments allowed by git-rev-list - :return: Iterator yielding Commit objects which are parents of self + Optional path or list of paths limiting the :class:`Commit`\s to those that + contain at least one of the paths. + + :param kwargs: + All arguments allowed by :manpage:`git-rev-list(1)`. + + :return: + Iterator yielding :class:`Commit` objects which are parents of ``self`` """ # skip ourselves skip = kwargs.get("skip", 1) @@ -335,41 +399,63 @@ def stats(self) -> Stats: """Create a git stat from changes between this commit and its first parent or from all changes done if this is the very first commit. - :return: git.Stats + :note: + If this commit is at the boundary of a shallow clone, this will + raise :exc:`~git.exc.GitCommandError`, since the parent object + was never fetched and only exists as a reference on this commit. + + :return: + :class:`Stats` """ - if not self.parents: - text = self.repo.git.diff_tree(self.hexsha, "--", numstat=True, no_renames=True, root=True) - text2 = "" - for line in text.splitlines()[1:]: + + def process_lines(lines: List[str]) -> str: + text = "" + for file_info, line in zip(lines, lines[len(lines) // 2 :]): + change_type = file_info.split("\t")[0][-1] (insertions, deletions, filename) = line.split("\t") - text2 += "%s\t%s\t%s\n" % (insertions, deletions, filename) - text = text2 + text += "%s\t%s\t%s\t%s\n" % (change_type, insertions, deletions, filename) + return text + + if not self.parents: + lines = self.repo.git.diff_tree( + self.hexsha, "--", numstat=True, no_renames=True, root=True, raw=True + ).splitlines()[1:] + text = process_lines(lines) else: - text = self.repo.git.diff(self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True) + lines = self.repo.git.diff( + self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True, raw=True + ).splitlines() + text = process_lines(lines) return Stats._list_from_string(self.repo, text) @property def trailers(self) -> Dict[str, str]: - """Get the trailers of the message as a dictionary + """Deprecated. Get the trailers of the message as a dictionary. - :note: This property is deprecated, please use either :attr:`trailers_list` or - :attr:`trailers_dict``. + :note: + This property is deprecated, please use either :attr:`trailers_list` or + :attr:`trailers_dict`. :return: - Dictionary containing whitespace stripped trailer information. Only contains - the latest instance of each trailer key. + Dictionary containing whitespace stripped trailer information. + Only contains the latest instance of each trailer key. """ + warnings.warn( + "Commit.trailers is deprecated, use Commit.trailers_list or Commit.trailers_dict instead", + DeprecationWarning, + stacklevel=2, + ) return {k: v[0] for k, v in self.trailers_dict.items()} @property def trailers_list(self) -> List[Tuple[str, str]]: """Get the trailers of the message as a list. - Git messages can contain trailer information that are similar to RFC 822 - e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers). + Git messages can contain trailer information that are similar to :rfc:`822` + e-mail headers. See :manpage:`git-interpret-trailers(1)`. - This functions calls ``git interpret-trailers --parse`` onto the message - to extract the trailer information, returns the raw trailer data as a list. + This function calls ``git interpret-trailers --parse`` onto the message to + extract the trailer information, returns the raw trailer data as a list. Valid message with trailer:: @@ -383,7 +469,6 @@ def trailers_list(self) -> List[Tuple[str, str]]: key1: value1.2 key2 : value 2 with inner spaces - Returned list will look like this:: [ @@ -392,14 +477,10 @@ def trailers_list(self) -> List[Tuple[str, str]]: ("key2", "value 2 with inner spaces"), ] - :return: List containing key-value tuples of whitespace stripped trailer information. """ - cmd = ["git", "interpret-trailers", "--parse"] - proc: Git.AutoInterrupt = self.repo.git.execute(cmd, as_process=True, istream=PIPE) # type: ignore - trailer: str = proc.communicate(str(self.message).encode())[0].decode("utf8") - trailer = trailer.strip() + trailer = self._interpret_trailers(self.repo, self.message, ["--parse"], encoding=self.encoding).strip() if not trailer: return [] @@ -411,16 +492,37 @@ def trailers_list(self) -> List[Tuple[str, str]]: return trailer_list + @classmethod + def _interpret_trailers( + cls, + repo: "Repo", + message: Union[str, bytes], + trailer_args: Sequence[str], + encoding: str = default_encoding, + ) -> str: + message_bytes = message if isinstance(message, bytes) else message.encode(encoding, errors="strict") + cmd = [repo.git.GIT_PYTHON_GIT_EXECUTABLE, "interpret-trailers", *trailer_args] + proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] + cmd, + as_process=True, + istream=PIPE, + ) + try: + stdout_bytes, _ = proc.communicate(message_bytes) + return stdout_bytes.decode(encoding, errors="strict") + finally: + finalize_process(proc) + @property def trailers_dict(self) -> Dict[str, List[str]]: """Get the trailers of the message as a dictionary. - Git messages can contain trailer information that are similar to RFC 822 - e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers). + Git messages can contain trailer information that are similar to :rfc:`822` + e-mail headers. See :manpage:`git-interpret-trailers(1)`. - This functions calls ``git interpret-trailers --parse`` onto the message - to extract the trailer information. The key value pairs are stripped of - leading and trailing whitespaces before they get saved into a dictionary. + This function calls ``git interpret-trailers --parse`` onto the message to + extract the trailer information. The key value pairs are stripped of leading and + trailing whitespaces before they get saved into a dictionary. Valid message with trailer:: @@ -434,7 +536,6 @@ def trailers_dict(self) -> Dict[str, List[str]]: key1: value1.2 key2 : value 2 with inner spaces - Returned dictionary will look like this:: { @@ -444,8 +545,8 @@ def trailers_dict(self) -> Dict[str, List[str]]: :return: - Dictionary containing whitespace stripped trailer information. - Mapping trailer keys to a list of their corresponding values. + Dictionary containing whitespace stripped trailer information, mapping + trailer keys to a list of their corresponding values. """ d = defaultdict(list) for key, val in self.trailers_list: @@ -454,13 +555,16 @@ def trailers_dict(self) -> Dict[str, List[str]]: @classmethod def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, IO]) -> Iterator["Commit"]: - """Parse out commit information into a list of Commit objects. + """Parse out commit information into a list of :class:`Commit` objects. - We expect one-line per commit, and parse the actual commit information directly + We expect one line per commit, and parse the actual commit information directly from our lighting fast object database. - :param proc: git-rev-list process instance - one sha per line - :return: iterator supplying :class:`Commit` objects + :param proc: + :manpage:`git-rev-list(1)` process instance - one sha per line. + + :return: + Iterator supplying :class:`Commit` objects """ # def is_proc(inp) -> TypeGuard[Popen]: @@ -471,11 +575,14 @@ def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, if hasattr(proc_or_stream, "wait"): proc_or_stream = cast(Popen, proc_or_stream) - if proc_or_stream.stdout is not None: - stream = proc_or_stream.stdout + stream = proc_or_stream.stdout + if stream is None: + raise ValueError("Process has no stdout stream") elif hasattr(proc_or_stream, "readline"): - proc_or_stream = cast(IO, proc_or_stream) # type: ignore [redundant-cast] + proc_or_stream = cast(IO, proc_or_stream) # type: ignore[redundant-cast] stream = proc_or_stream + else: + raise TypeError("Expected a process or stream") readline = stream.readline while True: @@ -492,8 +599,8 @@ def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, yield cls(repo, hex_to_bin(hexsha)) # END for each line in stream - # TODO: Review this - it seems process handling got a bit out of control - # due to many developers trying to fix the open file handles issue. + # TODO: Review this - it seems process handling got a bit out of control due to + # many developers trying to fix the open file handles issue. if hasattr(proc_or_stream, "wait"): proc_or_stream = cast(Popen, proc_or_stream) finalize_process(proc_or_stream) @@ -510,36 +617,61 @@ def create_from_tree( committer: Union[None, Actor] = None, author_date: Union[None, str, datetime.datetime] = None, commit_date: Union[None, str, datetime.datetime] = None, + trailers: Union[None, Dict[str, str], List[Tuple[str, str]]] = None, ) -> "Commit": - """Commit the given tree, creating a commit object. + """Commit the given tree, creating a :class:`Commit` object. + + :param repo: + :class:`~git.repo.base.Repo` object the commit should be part of. + + :param tree: + :class:`~git.objects.tree.Tree` object or hex or bin sha. + The tree of the new commit. + + :param message: + Commit message. It may be an empty string if no message is provided. It will + be converted to a string, in any case. - :param repo: Repo object the commit should be part of - :param tree: Tree object or hex or bin sha. The tree of the new commit. - :param message: Commit message. It may be an empty string if no message is - provided. It will be converted to a string, in any case. :param parent_commits: - Optional :class:`Commit` objects to use as parents for the new commit. - If empty list, the commit will have no parents at all and become - a root commit. - If None, the current head commit will be the parent of the - new commit object. + Optional :class:`Commit` objects to use as parents for the new commit. If + empty list, the commit will have no parents at all and become a root commit. + If ``None``, the current head commit will be the parent of the new commit + object. + :param head: - If True, the HEAD will be advanced to the new commit automatically. + If ``True``, the HEAD will be advanced to the new commit automatically. Otherwise the HEAD will remain pointing on the previous commit. This could lead to undesired results when diffing files. - :param author: The name of the author, optional. If unset, the repository - configuration is used to obtain this value. - :param committer: The name of the committer, optional. If unset, the - repository configuration is used to obtain this value. - :param author_date: The timestamp for the author field. - :param commit_date: The timestamp for the committer field. - :return: Commit object representing the new commit. + :param author: + The name of the author, optional. + If unset, the repository configuration is used to obtain this value. + + :param committer: + The name of the committer, optional. + If unset, the repository configuration is used to obtain this value. + + :param author_date: + The timestamp for the author field. + + :param commit_date: + The timestamp for the committer field. + + :param trailers: + Optional trailer key-value pairs to append to the commit message. + Can be a dictionary mapping trailer keys to values, or a list of + ``(key, value)`` tuples (useful when the same key appears multiple + times, e.g. multiple ``Signed-off-by`` trailers). Trailers are + appended using ``git interpret-trailers``. + See :manpage:`git-interpret-trailers(1)`. + + :return: + :class:`Commit` object representing the new commit. :note: - Additional information about the committer and Author are taken from the - environment or from the git configuration, see git-commit-tree for - more information. + Additional information about the committer and author are taken from the + environment or from the git configuration. See :manpage:`git-commit-tree(1)` + for more information. """ if parent_commits is None: try: @@ -596,12 +728,27 @@ def create_from_tree( if not isinstance(conf_encoding, str): raise TypeError("conf_encoding could not be coerced to str") - # If the tree is no object, make sure we create one - otherwise - # the created commit object is invalid. + # If the tree is no object, make sure we create one - otherwise the created + # commit object is invalid. if isinstance(tree, str): tree = repo.tree(tree) # END tree conversion + # APPLY TRAILERS + if trailers: + trailer_args: List[str] = [] + if isinstance(trailers, dict): + for key, val in trailers.items(): + trailer_args.append("--trailer") + trailer_args.append(f"{key}: {val}") + else: + for key, val in trailers: + trailer_args.append("--trailer") + trailer_args.append(f"{key}: {val}") + + message = cls._interpret_trailers(repo, str(message), trailer_args) + # END apply trailers + # CREATE NEW COMMIT new_commit = cls( repo, @@ -621,8 +768,8 @@ def create_from_tree( new_commit.binsha = cls._calculate_sha_(repo, new_commit) if head: - # Need late import here, importing git at the very beginning throws - # as well... + # Need late import here, importing git at the very beginning throws as + # well... import git.refs try: @@ -719,7 +866,8 @@ def _deserialize(self, stream: BytesIO) -> "Commit": # END for each parent line self.parents = tuple(self.parents) - # We don't know actual author encoding before we have parsed it, so keep the lines around. + # We don't know actual author encoding before we have parsed it, so keep the + # lines around. author_line = next_line committer_line = readline() @@ -731,7 +879,8 @@ def _deserialize(self, stream: BytesIO) -> "Commit": next_line = readline() # END skip mergetags - # Now we can have the encoding line, or an empty line followed by the optional message. + # Now we can have the encoding line, or an empty line followed by the optional + # message. self.encoding = self.default_encoding self.gpgsig = "" @@ -767,7 +916,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": self.author_tz_offset, ) = parse_actor_and_date(author_line.decode(self.encoding, "replace")) except UnicodeDecodeError: - log.error( + _logger.error( "Failed to decode author line '%s' using encoding %s", author_line, self.encoding, @@ -781,7 +930,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": self.committer_tz_offset, ) = parse_actor_and_date(committer_line.decode(self.encoding, "replace")) except UnicodeDecodeError: - log.error( + _logger.error( "Failed to decode committer line '%s' using encoding %s", committer_line, self.encoding, @@ -795,7 +944,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": try: self.message = self.message.decode(self.encoding, "replace") except UnicodeDecodeError: - log.error( + _logger.error( "Failed to decode message '%s' using encoding %s", self.message, self.encoding, @@ -809,19 +958,20 @@ def _deserialize(self, stream: BytesIO) -> "Commit": @property def co_authors(self) -> List[Actor]: - """ - Search the commit message for any co-authors of this commit. + """Search the commit message for any co-authors of this commit. - Details on co-authors: https://github.blog/2018-01-29-commit-together-with-co-authors/ + Details on co-authors: + https://github.blog/2018-01-29-commit-together-with-co-authors/ - :return: List of co-authors for this commit (as Actor objects). + :return: + List of co-authors for this commit (as :class:`~git.util.Actor` objects). """ co_authors = [] if self.message: results = re.findall( r"^Co-authored-by: (.*) <(.*?)>$", - self.message, + str(self.message), re.MULTILINE, ) for author in results: diff --git a/git/objects/fun.py b/git/objects/fun.py index bf6bc21d6..ad5fbd59b 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -3,8 +3,14 @@ """Functions that are supposed to be as fast as possible.""" -from stat import S_ISDIR +__all__ = [ + "tree_to_stream", + "tree_entries_from_data", + "traverse_trees_recursive", + "traverse_tree_recursive", +] +from stat import S_ISDIR from git.compat import safe_decode, defenc @@ -23,27 +29,23 @@ if TYPE_CHECKING: from _typeshed import ReadableBuffer + from git import GitCmdObjectDB -EntryTup = Tuple[bytes, int, str] # same as TreeCacheTup in tree.py +EntryTup = Tuple[bytes, int, str] # Same as TreeCacheTup in tree.py. EntryTupOrNone = Union[EntryTup, None] # --------------------------------------------------- -__all__ = ( - "tree_to_stream", - "tree_entries_from_data", - "traverse_trees_recursive", - "traverse_tree_recursive", -) - - def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer"], Union[int, None]]) -> None: - """Write the given list of entries into a stream using its write method. + """Write the given list of entries into a stream using its ``write`` method. - :param entries: **sorted** list of tuples with (binsha, mode, name) - :param write: write method which takes a data string + :param entries: + **Sorted** list of tuples with (binsha, mode, name). + + :param write: + A ``write`` method which takes a data string. """ ord_zero = ord("0") bit_mask = 7 # 3 bits set. @@ -59,11 +61,11 @@ def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer mode_str = mode_str[1:] # END save a byte - # Here it comes: If the name is actually unicode, the replacement below - # will not work as the binsha is not part of the ascii unicode encoding - - # hence we must convert to an UTF-8 string for it to work properly. - # According to my tests, this is exactly what git does, that is it just - # takes the input literally, which appears to be UTF-8 on linux. + # Here it comes: If the name is actually unicode, the replacement below will not + # work as the binsha is not part of the ascii unicode encoding - hence we must + # convert to an UTF-8 string for it to work properly. According to my tests, + # this is exactly what git does, that is it just takes the input literally, + # which appears to be UTF-8 on linux. if isinstance(name, str): name_bytes = name.encode(defenc) else: @@ -73,10 +75,15 @@ def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer def tree_entries_from_data(data: bytes) -> List[EntryTup]: - """Reads the binary representation of a tree and returns tuples of Tree items + """Read the binary representation of a tree and returns tuples of + :class:`~git.objects.tree.Tree` items. + + :param data: + Data block with tree data (as bytes). - :param data: data block with tree data (as bytes) - :return: list(tuple(binsha, mode, tree_relative_path), ...)""" + :return: + list(tuple(binsha, mode, tree_relative_path), ...) + """ ord_zero = ord("0") space_ord = ord(" ") len_data = len(data) @@ -89,8 +96,8 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: # Some git versions truncate the leading 0, some don't. # The type will be extracted from the mode later. while data[i] != space_ord: - # Move existing mode integer up one level being 3 bits - # and add the actual ordinal value of the character. + # Move existing mode integer up one level being 3 bits and add the actual + # ordinal value of the character. mode = (mode << 3) + (data[i] - ord_zero) i += 1 # END while reading mode @@ -108,11 +115,11 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: # Default encoding for strings in git is UTF-8. # Only use the respective unicode object if the byte stream was encoded. name_bytes = data[ns:i] - name = safe_decode(name_bytes) + name = safe_decode(bytes(name_bytes)) # Byte is NULL, get next 20. i += 1 - sha = data[i : i + 20] + sha = bytes(data[i : i + 20]) i = i + 20 out.append((sha, mode, name)) # END for each byte in data stream @@ -120,10 +127,10 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: def _find_by_name(tree_data: MutableSequence[EntryTupOrNone], name: str, is_dir: bool, start_at: int) -> EntryTupOrNone: - """Return data entry matching the given name and tree mode or None. + """Return data entry matching the given name and tree mode or ``None``. - Before the item is returned, the respective data item is set None in the - tree_data list to mark it done. + Before the item is returned, the respective data item is set None in the `tree_data` + list to mark it done. """ try: @@ -144,13 +151,11 @@ def _find_by_name(tree_data: MutableSequence[EntryTupOrNone], name: str, is_dir: @overload -def _to_full_path(item: None, path_prefix: str) -> None: - ... +def _to_full_path(item: None, path_prefix: str) -> None: ... @overload -def _to_full_path(item: EntryTup, path_prefix: str) -> EntryTup: - ... +def _to_full_path(item: EntryTup, path_prefix: str) -> EntryTup: ... def _to_full_path(item: EntryTupOrNone, path_prefix: str) -> EntryTupOrNone: @@ -164,7 +169,9 @@ def traverse_trees_recursive( odb: "GitCmdObjectDB", tree_shas: Sequence[Union[bytes, None]], path_prefix: str ) -> List[Tuple[EntryTupOrNone, ...]]: """ - :return: list of list with entries according to the given binary tree-shas. + :return: + List of list with entries according to the given binary tree-shas. + The result is encoded in a list of n tuple|None per blob/commit, (n == len(tree_shas)), where: @@ -172,16 +179,19 @@ def traverse_trees_recursive( * [1] == mode as int * [2] == path relative to working tree root - The entry tuple is None if the respective blob/commit did not - exist in the given tree. + The entry tuple is ``None`` if the respective blob/commit did not exist in the + given tree. - :param tree_shas: iterable of shas pointing to trees. All trees must - be on the same level. A tree-sha may be None in which case None. + :param tree_shas: + Iterable of shas pointing to trees. All trees must be on the same level. + A tree-sha may be ``None``, in which case ``None``. - :param path_prefix: a prefix to be added to the returned paths on this level, - set it '' for the first iteration. + :param path_prefix: + A prefix to be added to the returned paths on this level. + Set it ``""`` for the first iteration. - :note: The ordering of the returned items will be partially lost. + :note: + The ordering of the returned items will be partially lost. """ trees_data: List[List[EntryTupOrNone]] = [] @@ -198,8 +208,8 @@ def traverse_trees_recursive( out: List[Tuple[EntryTupOrNone, ...]] = [] - # Find all matching entries and recursively process them together if the match - # is a tree. If the match is a non-tree item, put it into the result. + # Find all matching entries and recursively process them together if the match is a + # tree. If the match is a non-tree item, put it into the result. # Processed items will be set None. for ti, tree_data in enumerate(trees_data): for ii, item in enumerate(tree_data): @@ -213,8 +223,8 @@ def traverse_trees_recursive( is_dir = S_ISDIR(mode) # Type mode bits # Find this item in all other tree data items. - # Wrap around, but stop one before our current index, hence - # ti+nt, not ti+1+nt. + # Wrap around, but stop one before our current index, hence ti+nt, not + # ti+1+nt. for tio in range(ti + 1, ti + nt): tio = tio % nt entries[tio] = _find_by_name(trees_data[tio], name, is_dir, ii) @@ -245,7 +255,8 @@ def traverse_trees_recursive( def traverse_tree_recursive(odb: "GitCmdObjectDB", tree_sha: bytes, path_prefix: str) -> List[EntryTup]: """ - :return: list of entries of the tree pointed to by the binary tree_sha. + :return: + List of entries of the tree pointed to by the binary `tree_sha`. An entry has the following format: @@ -253,7 +264,8 @@ def traverse_tree_recursive(odb: "GitCmdObjectDB", tree_sha: bytes, path_prefix: * [1] mode as int * [2] path relative to the repository - :param path_prefix: Prefix to prepend to the front of all returned paths. + :param path_prefix: + Prefix to prepend to the front of all returned paths. """ entries = [] data = tree_entries_from_data(odb.stream(tree_sha).read()) diff --git a/git/objects/submodule/__init__.py b/git/objects/submodule/__init__.py index b11b568f2..c0604e76f 100644 --- a/git/objects/submodule/__init__.py +++ b/git/objects/submodule/__init__.py @@ -1,5 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# NOTE: Cannot import anything here as the top-level __init__ has to handle -# our dependencies. +__all__ = ["Submodule", "UpdateProgress", "RootModule", "RootUpdateProgress"] + +from .base import Submodule, UpdateProgress +from .root import RootModule, RootUpdateProgress diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index c3d353141..39e912321 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1,13 +1,19 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["Submodule", "UpdateProgress"] + import gc from io import BytesIO import logging +import ntpath import os import os.path as osp +import shlex import stat +import sys import uuid +import urllib import git from git.cmd import Git @@ -24,6 +30,7 @@ from git.util import ( IterableList, RemoteProgress, + _to_relative_path, join_path_native, rmtree, to_native_path_linux, @@ -38,31 +45,43 @@ sm_section, ) - # typing ---------------------------------------------------------------------- -from typing import Callable, Dict, Mapping, Sequence, TYPE_CHECKING, cast -from typing import Any, Iterator, Union -from git.types import Commit_ish, Literal, PathLike, TBD +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Mapping, + Sequence, + TYPE_CHECKING, + Union, + cast, +) + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +from git.types import Commit_ish, PathLike, TBD if TYPE_CHECKING: from git.index import IndexFile + from git.objects.commit import Commit + from git.refs import Head, RemoteReference from git.repo import Repo - from git.refs import Head - # ----------------------------------------------------------------------------- -__all__ = ["Submodule", "UpdateProgress"] - - -log = logging.getLogger("git.objects.submodule.base") -log.addHandler(logging.NullHandler()) +_logger = logging.getLogger(__name__) class UpdateProgress(RemoteProgress): """Class providing detailed progress information to the caller who should - derive from it and implement the ``update(...)`` message.""" + derive from it and implement the + :meth:`update(...) ` message.""" CLONE, FETCH, UPDWKTREE = [1 << x for x in range(RemoteProgress._num_op_codes, RemoteProgress._num_op_codes + 3)] _num_op_codes: int = RemoteProgress._num_op_codes + 3 @@ -78,8 +97,8 @@ class UpdateProgress(RemoteProgress): # IndexObject comes via the util module. It's a 'hacky' fix thanks to Python's import -# mechanism, which causes plenty of trouble if the only reason for packages and -# modules is refactoring - subpackages shouldn't depend on parent packages. +# mechanism, which causes plenty of trouble if the only reason for packages and modules +# is refactoring - subpackages shouldn't depend on parent packages. class Submodule(IndexObject, TraversableIterableObj): """Implements access to a git submodule. They are special in that their sha represents a commit in the submodule's repository which is to be checked out @@ -98,8 +117,8 @@ class Submodule(IndexObject, TraversableIterableObj): k_default_mode = stat.S_IFDIR | stat.S_IFLNK """Submodule flags. Submodules are directories with link-status.""" - type: Literal["submodule"] = "submodule" # type: ignore - """This is a bogus type for base class compatibility.""" + type: Literal["submodule"] = "submodule" # type: ignore[assignment] + """This is a bogus type string for base class compatibility.""" __slots__ = ("_parent_commit", "_url", "_branch_path", "_name", "__weakref__") @@ -112,7 +131,7 @@ def __init__( mode: Union[int, None] = None, path: Union[PathLike, None] = None, name: Union[str, None] = None, - parent_commit: Union[Commit_ish, None] = None, + parent_commit: Union["Commit", None] = None, url: Union[str, None] = None, branch_path: Union[PathLike, None] = None, ) -> None: @@ -121,13 +140,24 @@ def __init__( We only document the parameters that differ from :class:`~git.objects.base.IndexObject`. - :param repo: Our parent repository. - :param binsha: Binary sha referring to a commit in the remote repository. See - the ``url`` parameter. - :param parent_commit: See :meth:`set_parent_commit`. - :param url: The URL to the remote repository which is the submodule. - :param branch_path: Full (relative) path to ref to checkout when cloning the - remote repository. + :param repo: + Our parent repository. + + :param binsha: + Binary sha referring to a commit in the remote repository. + See the `url` parameter. + + :param parent_commit: + The :class:`~git.objects.commit.Commit` whose tree is supposed to contain + the ``.gitmodules`` blob, or ``None`` to always point to the most recent + commit. See :meth:`set_parent_commit` for details. + + :param url: + The URL to the remote repository which is the submodule. + + :param branch_path: + Full repository-relative path to ref to checkout when cloning the remote + repository. """ super().__init__(repo, binsha, mode, path) self.size = 0 @@ -135,7 +165,6 @@ def __init__( if url is not None: self._url = url if branch_path is not None: - # assert isinstance(branch_path, str) self._branch_path = branch_path if name is not None: self._name = name @@ -164,7 +193,7 @@ def _set_cache_(self, attr: str) -> None: @classmethod def _get_intermediate_items(cls, item: "Submodule") -> IterableList["Submodule"]: - """:return: all the submodules of our module repository""" + """:return: All the submodules of our module repository""" try: return cls.list_items(item.module()) except InvalidGitRepositoryError: @@ -204,13 +233,15 @@ def __repr__(self) -> str: @classmethod def _config_parser( - cls, repo: "Repo", parent_commit: Union[Commit_ish, None], read_only: bool + cls, repo: "Repo", parent_commit: Union["Commit", None], read_only: bool ) -> SubmoduleConfigParser: """ - :return: Config Parser constrained to our submodule in read or write mode + :return: + Config parser constrained to our submodule in read or write mode - :raise IOError: If the .gitmodules file cannot be found, either locally or in - the repository at the given parent commit. Otherwise the exception would be + :raise IOError: + If the ``.gitmodules`` file cannot be found, either locally or in the + repository at the given parent commit. Otherwise the exception would be delayed until the first access of the config parser. """ parent_matches_head = True @@ -218,7 +249,8 @@ def _config_parser( try: parent_matches_head = repo.head.commit == parent_commit except ValueError: - # We are most likely in an empty repository, so the HEAD doesn't point to a valid ref. + # We are most likely in an empty repository, so the HEAD doesn't point + # to a valid ref. pass # END handle parent_commit fp_module: Union[str, BytesIO] @@ -239,7 +271,7 @@ def _config_parser( raise ValueError("Cannot write blobs of 'historical' submodule configurations") # END handle writes of historical submodules - return SubmoduleConfigParser(fp_module, read_only=read_only) + return SubmoduleConfigParser(fp_module, read_only=read_only, merge_includes=False) def _clear_cache(self) -> None: """Clear the possibly changed values.""" @@ -252,16 +284,20 @@ def _clear_cache(self) -> None: # END for each name to delete @classmethod - def _sio_modules(cls, parent_commit: Commit_ish) -> BytesIO: - """:return: Configuration file as BytesIO - we only access it through the respective blob's data""" + def _sio_modules(cls, parent_commit: "Commit") -> BytesIO: + """ + :return: + Configuration file as :class:`~io.BytesIO` - we only access it through the + respective blob's data + """ sio = BytesIO(parent_commit.tree[cls.k_modules_file].data_stream.read()) sio.name = cls.k_modules_file return sio def _config_parser_constrained(self, read_only: bool) -> SectionConstraint: - """:return: Config Parser constrained to our submodule in read or write mode""" + """:return: Config parser constrained to our submodule in read or write mode""" try: - pc: Union["Commit_ish", None] = self.parent_commit + pc = self.parent_commit except ValueError: pc = None # END handle empty parent repository @@ -269,8 +305,20 @@ def _config_parser_constrained(self, read_only: bool) -> SectionConstraint: parser.set_submodule(self) return SectionConstraint(parser, sm_section(self.name)) + @classmethod + def _validated_name(cls, name: str) -> str: + if ( + not name + or name.startswith(("/", "\\")) + or ntpath.splitdrive(name)[0] + or ".." in name.replace("\\", "/").split("/") + ): + raise ValueError("Invalid submodule name %r" % name) + return name + @classmethod def _module_abspath(cls, parent_repo: "Repo", path: PathLike, name: str) -> PathLike: + name = cls._validated_name(name) if cls._need_gitfile_submodules(parent_repo.git): return osp.join(parent_repo.git_dir, "modules", name) if parent_repo.working_tree_dir: @@ -289,23 +337,52 @@ def _clone_repo( **kwargs: Any, ) -> "Repo": """ - :return: Repo instance of newly cloned repository - :param repo: Our parent repository - :param url: URL to clone from - :param path: Repository - relative path to the submodule checkout location - :param name: Canonical name of the submodule - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :param kwargs: Additional arguments given to git.clone + :return: + :class:`~git.repo.base.Repo` instance of newly cloned repository. + + :param repo: + Our parent repository. + + :param url: + URL to clone from. + + :param path: + Repository-relative path to the submodule checkout location. + + :param name: + Canonical name of the submodule. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. + + :param kwargs: + Additional arguments given to :manpage:`git-clone(1)`. """ module_abspath = cls._module_abspath(repo, path, name) module_checkout_path = module_abspath if cls._need_gitfile_submodules(repo.git): + if not allow_unsafe_options: + Git.check_unsafe_options(Git._option_candidates([], kwargs), repo.unsafe_git_clone_options) + multi_options = kwargs.get("multi_options") + if multi_options: + Git.check_unsafe_options( + shlex.split(" ".join(cast("Sequence[str]", multi_options))), + repo.unsafe_git_clone_options, + ) + allow_unsafe_options = True kwargs["separate_git_dir"] = module_abspath module_abspath_dir = osp.dirname(module_abspath) if not osp.isdir(module_abspath_dir): os.makedirs(module_abspath_dir) - module_checkout_path = osp.join(str(repo.working_tree_dir), path) + module_checkout_path = osp.join(repo.working_tree_dir, path) # type: ignore[arg-type] + + if url.startswith("../"): + remote_name = cast("RemoteReference", repo.active_branch.tracking_branch()).remote_name + repo_remote_url = repo.remote(remote_name).url + url = os.path.join(repo_remote_url, url) clone = git.Repo.clone_from( url, @@ -321,44 +398,48 @@ def _clone_repo( @classmethod def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: - """:return: a path guaranteed to be relative to the given parent - repository - :raise ValueError: if path is not contained in the parent repository's working tree""" - path = to_native_path_linux(path) + """:return: A path guaranteed to be relative to the given parent repository + + :raise ValueError: + If path is not contained in the parent repository's working tree. + """ + if parent_repo.working_tree_dir: + path = _to_relative_path(parent_repo.working_tree_dir, path) + else: + path = to_native_path_linux(path) if path.endswith("/"): path = path[:-1] - # END handle trailing slash - - if osp.isabs(path) and parent_repo.working_tree_dir: - working_tree_linux = to_native_path_linux(parent_repo.working_tree_dir) - if not path.startswith(working_tree_linux): - raise ValueError( - "Submodule checkout path '%s' needs to be within the parents repository at '%s'" - % (working_tree_linux, path) - ) - path = path[len(working_tree_linux.rstrip("/")) + 1 :] - if not path: - raise ValueError("Absolute submodule path '%s' didn't yield a valid relative path" % path) - # END verify converted relative path makes sense - # END convert to a relative path + if not path or path == ".": + raise ValueError("Submodule checkout path must not be the repository root") return path @classmethod def _write_git_file_and_module_config(cls, working_tree_dir: PathLike, module_abspath: PathLike) -> None: - """Write a .git file containing a(preferably) relative path to the actual git module repository. + """Write a ``.git`` file containing a (preferably) relative path to the actual + git module repository. + + It is an error if the `module_abspath` cannot be made into a relative path, + relative to the `working_tree_dir`. + + :note: + This will overwrite existing files! + + :note: + As we rewrite both the git file as well as the module configuration, we + might fail on the configuration and will not roll back changes done to the + git file. This should be a non-issue, but may easily be fixed if it becomes + one. - It is an error if the module_abspath cannot be made into a relative path, relative to the working_tree_dir + :param working_tree_dir: + Directory to write the ``.git`` file into. - :note: This will overwrite existing files! - :note: as we rewrite both the git file as well as the module configuration, we might fail on the configuration - and will not roll back changes done to the git file. This should be a non - issue, but may easily be fixed - if it becomes one. - :param working_tree_dir: Directory to write the .git file into - :param module_abspath: Absolute path to the bare repository + :param module_abspath: + Absolute path to the bare repository. """ git_file = osp.join(working_tree_dir, ".git") rela_path = osp.relpath(module_abspath, start=working_tree_dir) - if os.name == "nt" and osp.isfile(git_file): + if sys.platform == "win32" and osp.isfile(git_file): os.remove(git_file) with open(git_file, "wb") as fp: fp.write(("gitdir: %s" % rela_path).encode(defenc)) @@ -387,55 +468,80 @@ def add( allow_unsafe_options: bool = False, allow_unsafe_protocols: bool = False, ) -> "Submodule": - """Add a new submodule to the given repository. This will alter the index - as well as the .gitmodules file, but will not create a new commit. - If the submodule already exists, no matter if the configuration differs - from the one provided, the existing submodule will be returned. - - :param repo: Repository instance which should receive the submodule. - :param name: The name/identifier for the submodule. - :param path: Repository-relative or absolute path at which the submodule - should be located. + """Add a new submodule to the given repository. This will alter the index as + well as the ``.gitmodules`` file, but will not create a new commit. If the + submodule already exists, no matter if the configuration differs from the one + provided, the existing submodule will be returned. + + :param repo: + Repository instance which should receive the submodule. + + :param name: + The name/identifier for the submodule. + + :param path: + Repository-relative or absolute path at which the submodule should be + located. It will be created as required during the repository initialization. - :param url: git-clone compatible URL, see git-clone reference for more information. - If None, the repository is assumed to exist, and the url of the first - remote is taken instead. This is useful if you want to make an existing - repository a submodule of another one. - :param branch: name of branch at which the submodule should (later) be checked out. - The given branch must exist in the remote repository, and will be checked - out locally as a tracking branch. - It will only be written into the configuration if it not None, which is + + :param url: + ``git clone ...``-compatible URL. See :manpage:`git-clone(1)` for more + information. If ``None``, the repository is assumed to exist, and the URL of + the first remote is taken instead. This is useful if you want to make an + existing repository a submodule of another one. + + :param branch: + Name of branch at which the submodule should (later) be checked out. The + given branch must exist in the remote repository, and will be checked out + locally as a tracking branch. + It will only be written into the configuration if it not ``None``, which is when the checked out branch will be the one the remote HEAD pointed to. - The result you get in these situation is somewhat fuzzy, and it is recommended - to specify at least 'master' here. - Examples are 'master' or 'feature/new'. - :param no_checkout: If True, and if the repository has to be cloned manually, - no checkout will be performed. - :param depth: Create a shallow clone with a history truncated to the - specified number of commits. - :param env: Optional dictionary containing the desired environment variables. + The result you get in these situation is somewhat fuzzy, and it is + recommended to specify at least ``master`` here. + Examples are ``master`` or ``feature/new``. + + :param no_checkout: + If ``True``, and if the repository has to be cloned manually, no checkout + will be performed. + + :param depth: + Create a shallow clone with a history truncated to the specified number of + commits. + + :param env: + Optional dictionary containing the desired environment variables. + Note: Provided variables will be used to update the execution environment for ``git``. If some variable is not specified in `env` and is defined in attr:`os.environ`, the value from attr:`os.environ` will be used. If you want to unset some variable, consider providing an empty string as its value. - :param clone_multi_options: A list of Clone options. Please see + + :param clone_multi_options: + A list of clone options. Please see :meth:`Repo.clone ` for details. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :return: The newly created submodule instance. - :note: Works atomically, such that no change will be done if the repository - update fails for instance. - """ + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. + + :return: + The newly created :class:`Submodule` instance. + + :note: + Works atomically, such that no change will be done if, for example, the + repository update fails. + """ if repo.bare: raise InvalidGitRepositoryError("Cannot add submodules to bare repositories") # END handle bare repos + cls._validated_name(name) path = cls._to_relative_path(repo, path) - # Ensure we never put backslashes into the URL, as some operating systems - # like it... + # Ensure we never put backslashes into the URL, as might happen on Windows. if url is not None: url = to_native_path_linux(url) # END ensure URL correctness @@ -452,7 +558,7 @@ def add( if sm.exists(): # Reretrieve submodule from tree. try: - sm = repo.head.commit.tree[str(path)] + sm = repo.head.commit.tree[os.fspath(path)] sm._name = name return sm except KeyError: @@ -518,7 +624,7 @@ def add( # END verify url ## See #525 for ensuring git URLs in config-files are valid under Windows. - url = Git.polish_url(url) + url = Git.polish_url(url, expand_vars=False) # It's important to add the URL to the parent config, to let `git submodule` know. # Otherwise there is a '-' character in front of the submodule listing: @@ -562,56 +668,75 @@ def update( allow_unsafe_options: bool = False, allow_unsafe_protocols: bool = False, ) -> "Submodule": - """Update the repository of this submodule to point to the checkout - we point at with the binsha of this instance. + """Update the repository of this submodule to point to the checkout we point at + with the binsha of this instance. :param recursive: - If True, we will operate recursively and update child modules as well. + If ``True``, we will operate recursively and update child modules as well. + :param init: - If True, the module repository will be cloned into place if necessary. + If ``True``, the module repository will be cloned into place if necessary. + :param to_latest_revision: - If True, the submodule's sha will be ignored during checkout. Instead, the - remote will be fetched, and the local tracking branch updated. This only + If ``True``, the submodule's sha will be ignored during checkout. Instead, + the remote will be fetched, and the local tracking branch updated. This only works if we have a local tracking branch, which is the case if the remote - repository had a master branch, or of the 'branch' option was specified for - this submodule and the branch existed remotely. + repository had a master branch, or if the ``branch`` option was specified + for this submodule and the branch existed remotely. + :param progress: - UpdateProgress instance or None if no progress should be shown. + :class:`UpdateProgress` instance, or ``None`` if no progress should be + shown. + :param dry_run: - If True, the operation will only be simulated, but not performed. + If ``True``, the operation will only be simulated, but not performed. All performed operations are read-only. + :param force: - If True, we may reset heads even if the repository in question is dirty. + If ``True``, we may reset heads even if the repository in question is dirty. Additionally we will be allowed to set a tracking branch which is ahead of its remote branch back into the past or the location of the remote branch. This will essentially 'forget' commits. - If False, local tracking branches that are in the future of their respective - remote branches will simply not be moved. + + If ``False``, local tracking branches that are in the future of their + respective remote branches will simply not be moved. + :param keep_going: - If True, we will ignore but log all errors, and keep going recursively. - Unless dry_run is set as well, keep_going could cause subsequent / inherited - errors you wouldn't see otherwise. - In conjunction with dry_run, it can be useful to anticipate all errors when - updating submodules. - :param env: Optional dictionary containing the desired environment variables. + If ``True``, we will ignore but log all errors, and keep going recursively. + Unless `dry_run` is set as well, `keep_going` could cause + subsequent/inherited errors you wouldn't see otherwise. + In conjunction with `dry_run`, it can be useful to anticipate all errors + when updating submodules. + + :param env: + Optional dictionary containing the desired environment variables. + Note: Provided variables will be used to update the execution environment for ``git``. If some variable is not specified in `env` and is defined in attr:`os.environ`, value from attr:`os.environ` will be used. + If you want to unset some variable, consider providing the empty string as its value. + :param clone_multi_options: - List of Clone options. + List of :manpage:`git-clone(1)` options. Please see :meth:`Repo.clone ` for details. They only take effect with the `init` option. + :param allow_unsafe_protocols: - Allow unsafe protocols to be used, like ext. + Allow unsafe protocols to be used, like ``ext``. + :param allow_unsafe_options: - Allow unsafe options to be used, like --upload-pack. + Allow unsafe options to be used, like ``--upload-pack``. - :note: Does nothing in bare repositories. - :note: This method is definitely not atomic if `recursive` is True. + :note: + Does nothing in bare repositories. - :return: self + :note: + This method is definitely not atomic if `recursive` is ``True``. + + :return: + self """ if self.repo.bare: return self @@ -625,130 +750,147 @@ def update( prefix = "DRY-RUN: " # END handle prefix - # To keep things plausible in dry-run mode. - if dry_run: - mrepo = None + mrepo = None # END init mrepo + def fetch_remotes(module_repo: "Repo") -> None: + rmts = module_repo.remotes + len_rmts = len(rmts) + for i, remote in enumerate(rmts): + op = FETCH + if i == 0: + op |= BEGIN + # END handle start + + progress.update( + op, + i, + len_rmts, + prefix + "Fetching remote %s of submodule %r" % (remote, self.name), + ) + # =============================== + if not dry_run: + remote.fetch(progress=progress) + # END handle dry-run + # =============================== + if i == len_rmts - 1: + op |= END + # END handle end + progress.update( + op, + i, + len_rmts, + prefix + "Done fetching remote of submodule %r" % self.name, + ) + # END fetch new data + try: + self._validated_name(self.name) + # ENSURE REPO IS PRESENT AND UP-TO-DATE ####################################### try: mrepo = self.module() - rmts = mrepo.remotes - len_rmts = len(rmts) - for i, remote in enumerate(rmts): - op = FETCH - if i == 0: - op |= BEGIN - # END handle start - - progress.update( - op, - i, - len_rmts, - prefix + "Fetching remote %s of submodule %r" % (remote, self.name), - ) - # =============================== - if not dry_run: - remote.fetch(progress=progress) - # END handle dry-run - # =============================== - if i == len_rmts - 1: - op |= END - # END handle end - progress.update( - op, - i, - len_rmts, - prefix + "Done fetching remote of submodule %r" % self.name, - ) - # END fetch new data + fetch_remotes(mrepo) except InvalidGitRepositoryError: mrepo = None if not init: return self # END early abort if init is not allowed - # There is no git-repository yet - but delete empty paths. checkout_module_abspath = self.abspath - if not dry_run and osp.isdir(checkout_module_abspath): - try: - os.rmdir(checkout_module_abspath) - except OSError as e: - raise OSError( - "Module directory at %r does already exist and is non-empty" % checkout_module_abspath - ) from e - # END handle OSError - # END handle directory removal - - # Don't check it out at first - nonetheless it will create a local - # branch according to the remote-HEAD if possible. - progress.update( - BEGIN | CLONE, - 0, - 1, - prefix - + "Cloning url '%s' to '%s' in submodule %r" % (self.url, checkout_module_abspath, self.name), - ) - if not dry_run: - mrepo = self._clone_repo( - self.repo, - self.url, - self.path, - self.name, - n=True, - env=env, - multi_options=clone_multi_options, - allow_unsafe_options=allow_unsafe_options, - allow_unsafe_protocols=allow_unsafe_protocols, - ) - # END handle dry-run - progress.update( - END | CLONE, - 0, - 1, - prefix + "Done cloning to %s" % checkout_module_abspath, - ) + module_abspath = self._module_abspath(self.repo, self.path, self.name) - if not dry_run: - # See whether we have a valid branch to check out. + # ``git submodule deinit`` leaves the repository in + # ``.git/modules`` and empties the checkout. Reconnect that retained + # repository instead of trying to clone over it. + if not dry_run and osp.isdir(module_abspath): try: - mrepo = cast("Repo", mrepo) - # Find a remote which has our branch - we try to be flexible. - remote_branch = find_first_remote_branch(mrepo.remotes, self.branch_name) - local_branch = mkhead(mrepo, self.branch_path) - - # Have a valid branch, but no checkout - make sure we can figure - # that out by marking the commit with a null_sha. - local_branch.set_object(Object(mrepo, self.NULL_BIN_SHA)) - # END initial checkout + branch creation - - # Make sure HEAD is not detached. - mrepo.head.set_reference( - local_branch, - logmsg="submodule: attaching head to %s" % local_branch, + git.Repo(module_abspath) + except InvalidGitRepositoryError: + pass + else: + if osp.lexists(checkout_module_abspath) and ( + osp.islink(checkout_module_abspath) + or not osp.isdir(checkout_module_abspath) + or os.listdir(checkout_module_abspath) + ): + raise OSError( + "Module directory at %r does already exist and is non-empty" % checkout_module_abspath + ) + os.makedirs(checkout_module_abspath, exist_ok=True) + self._write_git_file_and_module_config(checkout_module_abspath, module_abspath) + mrepo = git.Repo(checkout_module_abspath) + mrepo.head.reset(mrepo.head.commit, index=True, working_tree=True) + fetch_remotes(mrepo) + with self.repo.config_writer() as writer: + writer.set_value(sm_section(self.name), "url", self.url) + + if mrepo is None: + # There is no git-repository yet - but delete empty paths. + if not dry_run and osp.isdir(checkout_module_abspath): + try: + os.rmdir(checkout_module_abspath) + except OSError as e: + raise OSError( + "Module directory at %r does already exist and is non-empty" % checkout_module_abspath + ) from e + # END handle directory removal + + # Don't check it out at first - nonetheless it will create a local + # branch according to the remote-HEAD if possible. + progress.update( + BEGIN | CLONE, + 0, + 1, + prefix + + "Cloning url '%s' to '%s' in submodule %r" % (self.url, checkout_module_abspath, self.name), + ) + if not dry_run: + if self.url.startswith("."): + url = urllib.parse.urljoin(self.repo.remotes.origin.url + "/", self.url) + else: + url = self.url + mrepo = self._clone_repo( + self.repo, + url, + self.path, + self.name, + n=True, + env=env, + multi_options=clone_multi_options, + allow_unsafe_options=allow_unsafe_options, + allow_unsafe_protocols=allow_unsafe_protocols, ) - mrepo.head.reference.set_tracking_branch(remote_branch) - except (IndexError, InvalidGitRepositoryError): - log.warning("Failed to checkout tracking branch %s", self.branch_path) - # END handle tracking branch - - # NOTE: Have to write the repo config file as well, otherwise the - # default implementation will be offended and not update the repository. - # Maybe this is a good way to ensure it doesn't get into our way, but - # we want to stay backwards compatible too... It's so redundant! - with self.repo.config_writer() as writer: - writer.set_value(sm_section(self.name), "url", self.url) - # END handle dry_run + progress.update(END | CLONE, 0, 1, prefix + "Done cloning to %s" % checkout_module_abspath) + + if not dry_run: + # See whether we have a valid branch to check out. + try: + mrepo = cast("Repo", mrepo) + remote_branch = find_first_remote_branch(mrepo.remotes, self.branch_name) + local_branch = mkhead(mrepo, self.branch_path) + local_branch.set_object(Object(mrepo, self.NULL_BIN_SHA)) + mrepo.head.set_reference( + local_branch, + logmsg="submodule: attaching head to %s" % local_branch, + ) + mrepo.head.reference.set_tracking_branch(remote_branch) + except (IndexError, InvalidGitRepositoryError): + _logger.warning("Failed to checkout tracking branch %s", self.branch_path) + + with self.repo.config_writer() as writer: + writer.set_value(sm_section(self.name), "url", self.url) # END handle initialization # DETERMINE SHAS TO CHECK OUT ############################# binsha = self.binsha hexsha = self.hexsha + is_detached = False if mrepo is not None: - # mrepo is only set if we are not in dry-run mode or if the module existed. + # mrepo is only set if we are not in dry-run mode or if the module + # existed. is_detached = mrepo.head.is_detached # END handle dry_run @@ -761,24 +903,26 @@ def update( binsha = rcommit.binsha hexsha = rcommit.hexsha else: - log.error( + _logger.error( "%s a tracking branch was not set for local branch '%s'", msg_base, mrepo.head.reference, ) # END handle remote ref else: - log.error("%s there was no local tracking branch", msg_base) + _logger.error("%s there was no local tracking branch", msg_base) # END handle detached head # END handle to_latest_revision option # Update the working tree. # Handles dry_run. if mrepo is not None and mrepo.head.commit.binsha != binsha: - # We must ensure that our destination sha (the one to point to) is in the future of our current head. - # Otherwise, we will reset changes that might have been done on the submodule, but were not yet pushed. - # We also handle the case that history has been rewritten, leaving no merge-base. In that case - # we behave conservatively, protecting possible changes the user had done. + # We must ensure that our destination sha (the one to point to) is in + # the future of our current head. Otherwise, we will reset changes that + # might have been done on the submodule, but were not yet pushed. We + # also handle the case that history has been rewritten, leaving no + # merge-base. In that case we behave conservatively, protecting possible + # changes the user had done. may_reset = True if mrepo.head.commit.binsha != self.NULL_BIN_SHA: base_commit = mrepo.merge_base(mrepo.head.commit, hexsha) @@ -786,7 +930,7 @@ def update( if force: msg = "Will force checkout or reset on local branch that is possibly in the future of" msg += " the commit it will be checked out to, effectively 'forgetting' new commits" - log.debug(msg) + _logger.debug(msg) else: msg = "Skipping %s on branch '%s' of submodule repo '%s' as it contains un-pushed commits" msg %= ( @@ -794,7 +938,7 @@ def update( mrepo.head, mrepo, ) - log.info(msg) + _logger.info(msg) may_reset = False # END handle force # END handle if we are in the future @@ -815,10 +959,10 @@ def update( if not dry_run and may_reset: if is_detached: - # NOTE: For now we force. The user is not supposed to change detached - # submodules anyway. Maybe at some point this becomes an option, to - # properly handle user modifications - see below for future options - # regarding rebase and merge. + # NOTE: For now we force. The user is not supposed to change + # detached submodules anyway. Maybe at some point this becomes + # an option, to properly handle user modifications - see below + # for future options regarding rebase and merge. mrepo.git.checkout(hexsha, force=force) else: mrepo.head.reset(hexsha, index=True, working_tree=True) @@ -834,7 +978,7 @@ def update( except Exception as err: if not keep_going: raise - log.error(str(err)) + _logger.error(str(err)) # END handle keep_going # HANDLE RECURSION @@ -864,24 +1008,36 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = the repository at our current path, changing the configuration, as well as adjusting our index entry accordingly. - :param module_path: The path to which to move our module in the parent - repository's working tree, given as repository - relative or absolute path. - Intermediate directories will be created accordingly. If the path already - exists, it must be empty. Trailing (back)slashes are removed automatically. - :param configuration: If True, the configuration will be adjusted to let - the submodule point to the given path. - :param module: If True, the repository managed by this submodule - will be moved as well. If False, we don't move the submodule's checkout, - which may leave the parent repository in an inconsistent state. - :return: self - :raise ValueError: If the module path existed and was not empty, or was a file. - :note: Currently the method is not atomic, and it could leave the repository - in an inconsistent state if a sub-step fails for some reason. + :param module_path: + The path to which to move our module in the parent repository's working + tree, given as repository-relative or absolute path. Intermediate + directories will be created accordingly. If the path already exists, it must + be empty. Trailing (back)slashes are removed automatically. + + :param configuration: + If ``True``, the configuration will be adjusted to let the submodule point + to the given path. + + :param module: + If ``True``, the repository managed by this submodule will be moved as well. + If ``False``, we don't move the submodule's checkout, which may leave the + parent repository in an inconsistent state. + + :return: + self + + :raise ValueError: + If the module path existed and was not empty, or was a file. + + :note: + Currently the method is not atomic, and it could leave the repository in an + inconsistent state if a sub-step fails for some reason. """ if module + configuration < 1: raise ValueError("You must specify to move at least the module or the configuration of the submodule") # END handle input + self._validated_name(self.name) module_checkout_path = self._to_relative_path(self.repo, module_path) # VERIFY DESTINATION @@ -933,8 +1089,8 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = # END handle git file rewrite # END move physical module - # Rename the index entry - we have to manipulate the index directly as - # git-mv cannot be used on submodules... yeah. + # Rename the index entry - we have to manipulate the index directly as git-mv + # cannot be used on submodules... yeah. previous_sm_path = self.path try: if configuration: @@ -960,7 +1116,8 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = raise # END handle undo rename - # Auto-rename submodule if its name was 'default', that is, the checkout directory. + # Auto-rename submodule if its name was 'default', that is, the checkout + # directory. if previous_sm_path == self.name: self.rename(module_checkout_path) @@ -975,35 +1132,53 @@ def remove( dry_run: bool = False, ) -> "Submodule": """Remove this submodule from the repository. This will remove our entry - from the .gitmodules file and the entry in the .git/config file. - - :param module: If True, the checked out module we point to will be deleted as - well. If that module is currently on a commit outside any branch in the - remote, or if it is ahead of its tracking branch, or if there are modified - or untracked files in its working tree, then the removal will fail. In case - the removal of the repository fails for these reasons, the submodule status - will not have been altered. + from the ``.gitmodules`` file and the entry in the ``.git/config`` file. + + :param module: + If ``True``, the checked out module we point to will be deleted as well. If + that module is currently on a commit outside any branch in the remote, or if + it is ahead of its tracking branch, or if there are modified or untracked + files in its working tree, then the removal will fail. In case the removal + of the repository fails for these reasons, the submodule status will not + have been altered. + If this submodule has child modules of its own, these will be deleted prior to touching the direct submodule. - :param force: Enforces the deletion of the module even though it contains - modifications. This basically enforces a brute-force file system based - deletion. - :param configuration: If True, the submodule is deleted from the configuration, - otherwise it isn't. Although this should be enabled most of the time, this - flag enables you to safely delete the repository of your submodule. - :param dry_run: If True, we will not actually do anything, but throw the errors - we would usually throw. - :return: self - :note: Doesn't work in bare repositories. - :note: Doesn't work atomically, as failure to remove any part of the submodule - will leave an inconsistent state. - :raise InvalidGitRepositoryError: Thrown if the repository cannot be deleted. - :raise OSError: If directories or files could not be removed. + + :param force: + Enforces the deletion of the module even though it contains modifications. + This basically enforces a brute-force file system based deletion. + + :param configuration: + If ``True``, the submodule is deleted from the configuration, otherwise it + isn't. Although this should be enabled most of the time, this flag enables + you to safely delete the repository of your submodule. + + :param dry_run: + If ``True``, we will not actually do anything, but throw the errors we would + usually throw. + + :return: + self + + :note: + Doesn't work in bare repositories. + + :note: + Doesn't work atomically, as failure to remove any part of the submodule will + leave an inconsistent state. + + :raise git.exc.InvalidGitRepositoryError: + Thrown if the repository cannot be deleted. + + :raise OSError: + If directories or files could not be removed. """ if not (module or configuration): raise ValueError("Need to specify to delete at least the module, or the configuration") # END handle parameters + self._validated_name(self.name) # Recursively remove children of this submodule. nc = 0 for csm in self.children(): @@ -1012,8 +1187,9 @@ def remove( del csm if configuration and not dry_run and nc > 0: - # Ensure we don't leave the parent repository in a dirty state, and commit our changes. - # It's important for recursive, unforced, deletions to work as expected. + # Ensure we don't leave the parent repository in a dirty state, and commit + # our changes. It's important for recursive, unforced, deletions to work as + # expected. self.module().index.commit("Removed at least one of child-modules of '%s'" % self.name) # END handle recursion @@ -1024,8 +1200,9 @@ def remove( git_dir = mod.git_dir if force: # Take the fast lane and just delete everything in our module path. - # TODO: If we run into permission problems, we have a highly inconsistent - # state. Delete the .git folders last, start with the submodules first. + # TODO: If we run into permission problems, we have a highly + # inconsistent state. Delete the .git folders last, start with the + # submodules first. mp = self.abspath method: Union[None, Callable[[PathLike], None]] = None if osp.islink(mp): @@ -1055,6 +1232,7 @@ def remove( for remote in mod.remotes: num_branches_with_new_commits = 0 rrefs = remote.refs + rref = None for rref in rrefs: num_branches_with_new_commits += len(mod.git.cherry(rref)) != 0 # END for each remote ref @@ -1120,20 +1298,56 @@ def remove( return self - def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) -> "Submodule": + @unbare_repo + def deinit(self, force: bool = False) -> "Submodule": + """Run ``git submodule deinit`` on this submodule. + + This is a thin wrapper around ``git submodule deinit ``, + which unregisters the submodule (removes its entry from + ``.git/config`` and empties the working-tree directory) + without deleting the submodule from ``.gitmodules`` + or its checked-out repository under ``.git/modules/``. + A subsequent :meth:`update` will re-initialize the + submodule from the retained contents. + + :param force: + If ``True``, pass ``--force`` to ``git submodule deinit``. This + allows deinitialization even when the submodule's working tree has + local modifications that would otherwise block the command. + + :return: + self + + :note: + Doesn't work in bare repositories. + """ + args: List[str] = [] + if force: + args.append("--force") + args.extend(["--", str(self.path)]) + self.repo.git.submodule("deinit", *args) + return self + + def set_parent_commit(self, commit: Union[Commit_ish, str, None], check: bool = True) -> "Submodule": """Set this instance to use the given commit whose tree is supposed to - contain the .gitmodules blob. + contain the ``.gitmodules`` blob. :param commit: - Commit-ish reference pointing at the root_tree, or None to always point to - the most recent commit + Commit-ish reference pointing at the root tree, or ``None`` to always point + to the most recent commit. + :param check: - If True, relatively expensive checks will be performed to verify + If ``True``, relatively expensive checks will be performed to verify validity of the submodule. - :raise ValueError: If the commit's tree didn't contain the .gitmodules blob. + + :raise ValueError: + If the commit's tree didn't contain the ``.gitmodules`` blob. + :raise ValueError: If the parent commit didn't store this submodule under the current path. - :return: self + + :return: + self """ if commit is None: self._parent_commit = None @@ -1172,21 +1386,29 @@ def config_writer( self, index: Union["IndexFile", None] = None, write: bool = True ) -> SectionConstraint["SubmoduleConfigParser"]: """ - :return: A config writer instance allowing you to read and write the data - belonging to this submodule into the .gitmodules file. + :return: + A config writer instance allowing you to read and write the data belonging + to this submodule into the ``.gitmodules`` file. - :param index: If not None, an IndexFile instance which should be written. - Defaults to the index of the Submodule's parent repository. - :param write: If True, the index will be written each time a configuration - value changes. + :param index: + If not ``None``, an :class:`~git.index.base.IndexFile` instance which should + be written. Defaults to the index of the :class:`Submodule`'s parent + repository. - :note: The parameters allow for a more efficient writing of the index, - as you can pass in a modified index on your own, prevent automatic writing, - and write yourself once the whole operation is complete. + :param write: + If ``True``, the index will be written each time a configuration value changes. - :raise ValueError: If trying to get a writer on a parent_commit which does not - match the current head commit. - :raise IOError: If the .gitmodules file/blob could not be read + :note: + The parameters allow for a more efficient writing of the index, as you can + pass in a modified index on your own, prevent automatic writing, and write + yourself once the whole operation is complete. + + :raise ValueError: + If trying to get a writer on a parent_commit which does not match the + current head commit. + + :raise IOError: + If the ``.gitmodules`` file/blob could not be read. """ writer = self._config_parser_constrained(read_only=False) if index is not None: @@ -1201,22 +1423,27 @@ def rename(self, new_name: str) -> "Submodule": :note: This method takes care of renaming the submodule in various places, such as: - * $parent_git_dir / config - * $working_tree_dir / .gitmodules + * ``$parent_git_dir / config`` + * ``$working_tree_dir / .gitmodules`` * (git >= v1.8.0: move submodule repository to new name) - As .gitmodules will be changed, you would need to make a commit afterwards. The - changed .gitmodules file will already be added to the index. + As ``.gitmodules`` will be changed, you would need to make a commit afterwards. + The changed ``.gitmodules`` file will already be added to the index. - :return: This submodule instance + :return: + This :class:`Submodule` instance """ if self.name == new_name: return self + self._validated_name(self.name) + self._validated_name(new_name) + # .git/config with self.repo.config_writer() as pw: - # As we ourselves didn't write anything about submodules into the parent .git/config, - # we will not require it to exist, and just ignore missing entries. + # As we ourselves didn't write anything about submodules into the parent + # .git/config, we will not require it to exist, and just ignore missing + # entries. if pw.has_section(sm_section(self.name)): pw.rename_section(sm_section(self.name), sm_section(new_name)) @@ -1251,11 +1478,15 @@ def rename(self, new_name: str) -> "Submodule": @unbare_repo def module(self) -> "Repo": """ - :return: Repo instance initialized from the repository at our submodule path + :return: + :class:`~git.repo.base.Repo` instance initialized from the repository at our + submodule path - :raise InvalidGitRepositoryError: If a repository was not available. This could - also mean that it was not yet initialized. + :raise git.exc.InvalidGitRepositoryError: + If a repository was not available. + This could also mean that it was not yet initialized. """ + self._validated_name(self.name) module_checkout_abspath = self.abspath try: repo = git.Repo(module_checkout_abspath) @@ -1269,7 +1500,11 @@ def module(self) -> "Repo": # END handle exceptions def module_exists(self) -> bool: - """:return: True if our module exists and is a valid git repository. See module() method.""" + """ + :return: + ``True`` if our module exists and is a valid git repository. + See the :meth:`module` method. + """ try: self.module() return True @@ -1279,9 +1514,10 @@ def module_exists(self) -> bool: def exists(self) -> bool: """ - :return: True if the submodule exists, False otherwise. Please note that - a submodule may exist (in the .gitmodules file) even though its module - doesn't exist on disk. + :return: + ``True`` if the submodule exists, ``False`` otherwise. + Please note that a submodule may exist (in the ``.gitmodules`` file) even + though its module doesn't exist on disk. """ # Keep attributes for later, and restore them if we have no valid data. # This way we do not actually alter the state of the object. @@ -1292,14 +1528,15 @@ def exists(self) -> bool: loc[attr] = getattr(self, attr) # END if we have the attribute cache except (cp.NoSectionError, ValueError): - # On PY3, this can happen apparently... don't know why this doesn't happen on PY2. + # On PY3, this can happen apparently... don't know why this doesn't + # happen on PY2. pass # END for each attr self._clear_cache() try: try: - self.path + self.path # noqa: B018 return True except Exception: return False @@ -1315,38 +1552,47 @@ def exists(self) -> bool: @property def branch(self) -> "Head": """ - :return: The branch instance that we are to checkout + :return: + The branch instance that we are to checkout - :raise InvalidGitRepositoryError: If our module is not yet checked out + :raise git.exc.InvalidGitRepositoryError: + If our module is not yet checked out. """ return mkhead(self.module(), self._branch_path) @property def branch_path(self) -> PathLike: """ - :return: Full (relative) path as string to the branch we would checkout - from the remote and track + :return: + Full repository-relative path as string to the branch we would checkout from + the remote and track """ return self._branch_path @property def branch_name(self) -> str: - """:return: The name of the branch, which is the shortest possible branch name""" - # Use an instance method, for this we create a temporary Head instance - # which uses a repository that is available at least (it makes no difference). + """ + :return: + The name of the branch, which is the shortest possible branch name + """ + # Use an instance method, for this we create a temporary Head instance which + # uses a repository that is available at least (it makes no difference). return git.Head(self.repo, self._branch_path).name @property def url(self) -> str: - """:return: The url to the repository which our module - repository refers to""" + """:return: The url to the repository our submodule's repository refers to""" return self._url @property - def parent_commit(self) -> "Commit_ish": + def parent_commit(self) -> "Commit": """ - :return: Commit instance with the tree containing the .gitmodules file + :return: + :class:`~git.objects.commit.Commit` instance with the tree containing the + ``.gitmodules`` file - :note: Will always point to the current head's commit if it was not set explicitly. + :note: + Will always point to the current head's commit if it was not set explicitly. """ if self._parent_commit is None: return self.repo.commit() @@ -1355,33 +1601,41 @@ def parent_commit(self) -> "Commit_ish": @property def name(self) -> str: """ - :return: The name of this submodule. It is used to identify it within the - .gitmodules file. + :return: + The name of this submodule. It is used to identify it within the + ``.gitmodules`` file. - :note: By default, this is the name is the path at which to find the submodule, - but in GitPython it should be a unique identifier similar to the identifiers + :note: + By default, this is the name is the path at which to find the submodule, but + in GitPython it should be a unique identifier similar to the identifiers used for remotes, which allows to change the path of the submodule easily. """ return self._name def config_reader(self) -> SectionConstraint[SubmoduleConfigParser]: """ - :return: ConfigReader instance which allows you to query the configuration - values of this submodule, as provided by the .gitmodules file. + :return: + ConfigReader instance which allows you to query the configuration values of + this submodule, as provided by the ``.gitmodules`` file. - :note: The config reader will actually read the data directly from the - repository and thus does not need nor care about your working tree. + :note: + The config reader will actually read the data directly from the repository + and thus does not need nor care about your working tree. - :note: Should be cached by the caller and only kept as long as needed. + :note: + Should be cached by the caller and only kept as long as needed. - :raise IOError: If the .gitmodules file/blob could not be read. + :raise IOError: + If the ``.gitmodules`` file/blob could not be read. """ return self._config_parser_constrained(read_only=True) def children(self) -> IterableList["Submodule"]: """ - :return: IterableList(Submodule, ...) an iterable list of submodules instances - which are children of this submodule or 0 if the submodule is not checked out. + :return: + IterableList(Submodule, ...) An iterable list of :class:`Submodule` + instances which are children of this submodule or 0 if the submodule is not + checked out. """ return self._get_intermediate_items(self) @@ -1394,10 +1648,14 @@ def iter_items( cls, repo: "Repo", parent_commit: Union[Commit_ish, str] = "HEAD", - *Args: Any, + *args: Any, **kwargs: Any, ) -> Iterator["Submodule"]: - """:return: Iterator yielding Submodule instances available in the given repository""" + """ + :return: + Iterator yielding :class:`Submodule` instances available in the given + repository + """ try: pc = repo.commit(parent_commit) # Parent commit instance parser = cls._config_parser(repo, pc, read_only=True) @@ -1425,8 +1683,8 @@ def iter_items( entry = index.entries[index.entry_key(p, 0)] sm = Submodule(repo, entry.binsha, entry.mode, entry.path) except KeyError: - # The submodule doesn't exist, probably it wasn't - # removed from the .gitmodules file. + # The submodule doesn't exist, probably it wasn't removed from the + # .gitmodules file. continue # END handle keyerror # END handle critical error diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index d9d9f6d24..d068049c1 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -1,13 +1,17 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from .base import Submodule, UpdateProgress -from .util import find_first_remote_branch -from git.exc import InvalidGitRepositoryError -import git +__all__ = ["RootModule", "RootUpdateProgress"] import logging +import git +from git.exc import InvalidGitRepositoryError +from git.util import IterableList + +from .base import Submodule, UpdateProgress +from .util import find_first_remote_branch + # typing ------------------------------------------------------------------- from typing import TYPE_CHECKING, Union @@ -16,18 +20,15 @@ if TYPE_CHECKING: from git.repo import Repo - from git.util import IterableList # ---------------------------------------------------------------------------- -__all__ = ["RootModule", "RootUpdateProgress"] - -log = logging.getLogger("git.objects.submodule.root") -log.addHandler(logging.NullHandler()) +_logger = logging.getLogger(__name__) class RootUpdateProgress(UpdateProgress): - """Utility class which adds more opcodes to the UpdateProgress.""" + """Utility class which adds more opcodes to + :class:`~git.objects.submodule.base.UpdateProgress`.""" REMOVE, PATHCHANGE, BRANCHCHANGE, URLCHANGE = [ 1 << x for x in range(UpdateProgress._num_op_codes, UpdateProgress._num_op_codes + 4) @@ -56,7 +57,7 @@ class RootModule(Submodule): k_root_name = "__ROOT__" - def __init__(self, repo: "Repo"): + def __init__(self, repo: "Repo") -> None: # repo, binsha, mode=None, path=None, name = None, parent_commit=None, url=None, ref=None) super().__init__( repo, @@ -75,9 +76,9 @@ def _clear_cache(self) -> None: # { Interface - def update( + def update( # type: ignore[override] self, - previous_commit: Union[Commit_ish, None] = None, # type: ignore[override] + previous_commit: Union[Commit_ish, str, None] = None, recursive: bool = True, force_remove: bool = False, init: bool = True, @@ -89,46 +90,64 @@ def update( ) -> "RootModule": """Update the submodules of this repository to the current HEAD commit. - This method behaves smartly by determining changes of the path of a submodules + This method behaves smartly by determining changes of the path of a submodule's repository, next to changes to the to-be-checked-out commit or the branch to be - checked out. This works if the submodules ID does not change. + checked out. This works if the submodule's ID does not change. + Additionally it will detect addition and removal of submodules, which will be handled gracefully. - :param previous_commit: If set to a commit-ish, the commit we should use as the - previous commit the HEAD pointed to before it was set to the commit it - points to now. - If None, it defaults to ``HEAD@{1}`` otherwise - :param recursive: if True, the children of submodules will be updated as well - using the same technique. - :param force_remove: If submodules have been deleted, they will be forcibly - removed. Otherwise the update may fail if a submodule's repository cannot be - deleted as changes have been made to it. + :param previous_commit: + If set to a commit-ish, the commit we should use as the previous commit the + HEAD pointed to before it was set to the commit it points to now. + If ``None``, it defaults to ``HEAD@{1}`` otherwise. + + :param recursive: + If ``True``, the children of submodules will be updated as well using the + same technique. + + :param force_remove: + If submodules have been deleted, they will be forcibly removed. Otherwise + the update may fail if a submodule's repository cannot be deleted as changes + have been made to it. (See :meth:`Submodule.update ` for more information.) - :param init: If we encounter a new module which would need to be initialized, - then do it. - :param to_latest_revision: If True, instead of checking out the revision pointed - to by this submodule's sha, the checked out tracking branch will be merged - with the latest remote branch fetched from the repository's origin. + + :param init: + If we encounter a new module which would need to be initialized, then do it. + + :param to_latest_revision: + If ``True``, instead of checking out the revision pointed to by this + submodule's sha, the checked out tracking branch will be merged with the + latest remote branch fetched from the repository's origin. + Unless `force_reset` is specified, a local tracking branch will never be reset into its past, therefore the remote branch must be in the future for this to have an effect. - :param force_reset: If True, submodules may checkout or reset their branch even - if the repository has pending changes that would be overwritten, or if the - local tracking branch is in the future of the remote tracking branch and - would be reset into its past. - :param progress: :class:`RootUpdateProgress` instance or None if no progress - should be sent. - :param dry_run: If True, operations will not actually be performed. Progress - messages will change accordingly to indicate the WOULD DO state of the - operation. - :param keep_going: If True, we will ignore but log all errors, and keep going - recursively. Unless `dry_run` is set as well, `keep_going` could cause + + :param force_reset: + If ``True``, submodules may checkout or reset their branch even if the + repository has pending changes that would be overwritten, or if the local + tracking branch is in the future of the remote tracking branch and would be + reset into its past. + + :param progress: + :class:`RootUpdateProgress` instance, or ``None`` if no progress should be + sent. + + :param dry_run: + If ``True``, operations will not actually be performed. Progress messages + will change accordingly to indicate the WOULD DO state of the operation. + + :param keep_going: + If ``True``, we will ignore but log all errors, and keep going recursively. + Unless `dry_run` is set as well, `keep_going` could cause subsequent/inherited errors you wouldn't see otherwise. In conjunction with `dry_run`, this can be useful to anticipate all errors when updating submodules. - :return: self + + :return: + self """ if self.repo.bare: raise InvalidGitRepositoryError("Cannot update submodules in bare repositories") @@ -143,6 +162,7 @@ def update( prefix = "DRY-RUN: " repo = self.repo + sms: "IterableList[Submodule]" = IterableList("name") try: # SETUP BASE COMMIT @@ -163,7 +183,7 @@ def update( # END handle previous commit psms: "IterableList[Submodule]" = self.list_items(repo, parent_commit=previous_commit) - sms: "IterableList[Submodule]" = self.list_items(repo) + sms = self.list_items(repo) spsms = set(psms) ssms = set(sms) @@ -235,13 +255,14 @@ def update( ################### if sm.url != psm.url: # Add the new remote, remove the old one. - # This way, if the url just changes, the commits will not - # have to be re-retrieved. + # This way, if the url just changes, the commits will not have + # to be re-retrieved. nn = "__new_origin__" smm = sm.module() rmts = smm.remotes - # Don't do anything if we already have the url we search in place. + # Don't do anything if we already have the url we search in + # place. if len([r for r in rmts if r.url == sm.url]) == 0: progress.update( BEGIN | URLCHANGE, @@ -273,17 +294,17 @@ def update( # END if urls match # END for each remote - # If we didn't find a matching remote, but have exactly one, - # we can safely use this one. + # If we didn't find a matching remote, but have exactly + # one, we can safely use this one. if rmt_for_deletion is None: if len(rmts) == 1: rmt_for_deletion = rmts[0] else: - # If we have not found any remote with the original URL - # we may not have a name. This is a special case, - # and its okay to fail here. - # Alternatively we could just generate a unique name and - # leave all existing ones in place. + # If we have not found any remote with the + # original URL we may not have a name. This is a + # special case, and its okay to fail here. + # Alternatively we could just generate a unique + # name and leave all existing ones in place. raise InvalidGitRepositoryError( "Couldn't find original remote-repo at url %r" % psm.url ) @@ -293,19 +314,19 @@ def update( orig_name = rmt_for_deletion.name smm.delete_remote(rmt_for_deletion) # NOTE: Currently we leave tags from the deleted remotes - # as well as separate tracking branches in the possibly totally - # changed repository (someone could have changed the url to - # another project). At some point, one might want to clean - # it up, but the danger is high to remove stuff the user - # has added explicitly. + # as well as separate tracking branches in the possibly + # totally changed repository (someone could have changed + # the url to another project). At some point, one might + # want to clean it up, but the danger is high to remove + # stuff the user has added explicitly. # Rename the new remote back to what it was. smr.rename(orig_name) - # Early on, we verified that the our current tracking branch - # exists in the remote. Now we have to ensure that the - # sha we point to is still contained in the new remote - # tracking branch. + # Early on, we verified that the our current tracking + # branch exists in the remote. Now we have to ensure + # that the sha we point to is still contained in the new + # remote tracking branch. smsha = sm.binsha found = False rref = smr.refs[self.branch_name] @@ -317,11 +338,12 @@ def update( # END for each commit if not found: - # Adjust our internal binsha to use the one of the remote - # this way, it will be checked out in the next step. - # This will change the submodule relative to us, so - # the user will be able to commit the change easily. - log.warning( + # Adjust our internal binsha to use the one of the + # remote this way, it will be checked out in the + # next step. This will change the submodule relative + # to us, so the user will be able to commit the + # change easily. + _logger.warning( "Current sha %s was not contained in the tracking\ branch at the new remote, setting it the the remote's tracking branch", sm.hexsha, @@ -329,7 +351,8 @@ def update( sm.binsha = rref.commit.binsha # END reset binsha - # NOTE: All checkout is performed by the base implementation of update. + # NOTE: All checkout is performed by the base + # implementation of update. # END handle dry_run progress.update( END | URLCHANGE, @@ -343,7 +366,8 @@ def update( # HANDLE PATH CHANGES ##################### if sm.branch_path != psm.branch_path: - # Finally, create a new tracking branch which tracks the new remote branch. + # Finally, create a new tracking branch which tracks the new + # remote branch. progress.update( BEGIN | BRANCHCHANGE, i, @@ -355,8 +379,8 @@ def update( if not dry_run: smm = sm.module() smmr = smm.remotes - # As the branch might not exist yet, we will have to fetch all remotes - # to be sure... + # As the branch might not exist yet, we will have to fetch + # all remotes to be sure... for remote in smmr: remote.fetch(progress=progress) # END for each remote @@ -373,11 +397,12 @@ def update( # END ensure tracking branch exists tbr.set_tracking_branch(find_first_remote_branch(smmr, sm.branch_name)) - # NOTE: All head-resetting is done in the base implementation of update - # but we will have to checkout the new branch here. As it still points - # to the currently checked out commit, we don't do any harm. - # As we don't want to update working-tree or index, changing the ref is - # all there is to do. + # NOTE: All head-resetting is done in the base + # implementation of update but we will have to checkout the + # new branch here. As it still points to the currently + # checked out commit, we don't do any harm. + # As we don't want to update working-tree or index, changing + # the ref is all there is to do. smm.head.reference = tbr # END handle dry_run @@ -393,7 +418,7 @@ def update( except Exception as err: if not keep_going: raise - log.error(str(err)) + _logger.error(str(err)) # END handle keep_going # FINALLY UPDATE ALL ACTUAL SUBMODULES @@ -410,10 +435,10 @@ def update( keep_going=keep_going, ) - # Update recursively depth first - question is which inconsistent - # state will be better in case it fails somewhere. Defective branch - # or defective depth. The RootSubmodule type will never process itself, - # which was done in the previous expression. + # Update recursively depth first - question is which inconsistent state will + # be better in case it fails somewhere. Defective branch or defective depth. + # The RootSubmodule type will never process itself, which was done in the + # previous expression. if recursive: # The module would exist by now if we are not in dry_run mode. if sm.module_exists(): diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index f8265798d..c021510d8 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -1,12 +1,20 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -import git -from git.exc import InvalidGitRepositoryError -from git.config import GitConfigParser +__all__ = [ + "sm_section", + "sm_name", + "mkhead", + "find_first_remote_branch", + "SubmoduleConfigParser", +] + from io import BytesIO import weakref +import git +from git.config import GitConfigParser +from git.exc import InvalidGitRepositoryError # typing ----------------------------------------------------------------------- @@ -15,27 +23,19 @@ from git.types import PathLike if TYPE_CHECKING: - from .base import Submodule from weakref import ReferenceType - from git.repo import Repo - from git.refs import Head - from git import Remote - from git.refs import RemoteReference + from git.refs import Head, RemoteReference + from git.remote import Remote + from git.repo import Repo -__all__ = ( - "sm_section", - "sm_name", - "mkhead", - "find_first_remote_branch", - "SubmoduleConfigParser", -) + from .base import Submodule # { Utilities def sm_section(name: str) -> str: - """:return: Section title used in .gitmodules configuration file""" + """:return: Section title used in ``.gitmodules`` configuration file""" return f'submodule "{name}"' @@ -51,7 +51,8 @@ def mkhead(repo: "Repo", path: PathLike) -> "Head": def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> "RemoteReference": - """Find the remote branch matching the name of the given branch or raise InvalidGitRepositoryError.""" + """Find the remote branch matching the name of the given branch or raise + :exc:`~git.exc.InvalidGitRepositoryError`.""" for remote in remotes: try: return remote.refs[branch_name] @@ -64,16 +65,16 @@ def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> " # } END utilities - # { Classes class SubmoduleConfigParser(GitConfigParser): - """Catches calls to _write, and updates the .gitmodules blob in the index - with the new data, if we have written into a stream. + """Catches calls to :meth:`~git.config.GitConfigParser.write`, and updates the + ``.gitmodules`` blob in the index with the new data, if we have written into a + stream. - Otherwise it would add the local file to the index to make it correspond - with the working tree. Additionally, the cache must be cleared. + Otherwise it would add the local file to the index to make it correspond with the + working tree. Additionally, the cache must be cleared. Please note that no mutating method will work in bare mode. """ @@ -86,8 +87,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # { Interface def set_submodule(self, submodule: "Submodule") -> None: - """Set this instance's submodule. It must be called before - the first write operation begins.""" + """Set this instance's submodule. It must be called before the first write + operation begins.""" self._smref = weakref.ref(submodule) def flush_to_index(self) -> None: diff --git a/git/objects/tag.py b/git/objects/tag.py index f7aaaf477..88671d316 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -3,29 +3,48 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Object-based types.""" +"""Provides an :class:`~git.objects.base.Object`-based type for annotated tags. + +This defines the :class:`TagObject` class, which represents annotated tags. +For lightweight tags, see the :mod:`git.refs.tag` module. +""" + +__all__ = ["TagObject"] + +import sys + +from git.compat import defenc +from git.util import Actor, hex_to_bin from . import base from .util import get_object_type_by_name, parse_actor_and_date -from ..util import hex_to_bin -from ..compat import defenc + +# typing ---------------------------------------------- from typing import List, TYPE_CHECKING, Union -from git.types import Literal +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal if TYPE_CHECKING: from git.repo import Repo - from git.util import Actor - from .commit import Commit + from .blob import Blob + from .commit import Commit from .tree import Tree -__all__ = ("TagObject",) +# --------------------------------------------------- class TagObject(base.Object): - """Non-lightweight tag carrying additional information about an object we are pointing to.""" + """Annotated (i.e. non-lightweight) tag carrying additional information about an + object we are pointing to. + + See :manpage:`gitglossary(7)` on "tag object": + https://git-scm.com/docs/gitglossary#def_tag_object + """ type: Literal["tag"] = "tag" @@ -44,24 +63,35 @@ def __init__( binsha: bytes, object: Union[None, base.Object] = None, tag: Union[None, str] = None, - tagger: Union[None, "Actor"] = None, + tagger: Union[None, Actor] = None, tagged_date: Union[int, None] = None, tagger_tz_offset: Union[int, None] = None, message: Union[str, None] = None, ) -> None: # @ReservedAssignment """Initialize a tag object with additional data. - :param repo: Repository this object is located in - :param binsha: 20 byte SHA1 - :param object: Object instance of object we are pointing to - :param tag: Name of this tag - :param tagger: Actor identifying the tagger + :param repo: + Repository this object is located in. + + :param binsha: + 20 byte SHA1. + + :param object: + :class:`~git.objects.base.Object` instance of object we are pointing to. + + :param tag: + Name of this tag. + + :param tagger: + :class:`~git.util.Actor` identifying the tagger. + :param tagged_date: int_seconds_since_epoch - The :class:`DateTime` of the tag creation. + The DateTime of the tag creation. Use :func:`time.gmtime` to convert it into a different format. - :param tagged_tz_offset: int_seconds_west_of_utc - The timezone that the authored_date is in, in a format similar - to :attr:`time.altzone`. + + :param tagger_tz_offset: int_seconds_west_of_utc + The timezone that the `tagged_date` is in, in a format similar to + :attr:`time.altzone`. """ super().__init__(repo, binsha) if object is not None: diff --git a/git/objects/tree.py b/git/objects/tree.py index a08adf48b..a3d611c80 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -3,17 +3,19 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from git.util import IterableList, join_path +__all__ = ["TreeModifier", "Tree"] + +import os +import sys + import git.diff as git_diff -from git.util import to_bin_sha +from git.util import IterableList, join_path, to_bin_sha from . import util -from .base import IndexObject, IndexObjUnion +from .base import IndexObjUnion, IndexObject from .blob import Blob -from .submodule.base import Submodule - from .fun import tree_entries_from_data, tree_to_stream - +from .submodule.base import Submodule # typing ------------------------------------------------- @@ -25,87 +27,42 @@ Iterator, List, Tuple, + TYPE_CHECKING, Type, Union, cast, - TYPE_CHECKING, ) -from git.types import PathLike, Literal +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +from git.types import PathLike if TYPE_CHECKING: - from git.repo import Repo from io import BytesIO + from git.repo import Repo + TreeCacheTup = Tuple[bytes, int, str] TraversedTreeTup = Union[Tuple[Union["Tree", None], IndexObjUnion, Tuple["Submodule", "Submodule"]]] - -# def is_tree_cache(inp: Tuple[bytes, int, str]) -> TypeGuard[TreeCacheTup]: -# return isinstance(inp[0], bytes) and isinstance(inp[1], int) and isinstance([inp], str) - # -------------------------------------------------------- -cmp: Callable[[str, str], int] = lambda a, b: (a > b) - (a < b) - -__all__ = ("TreeModifier", "Tree") - - -def git_cmp(t1: TreeCacheTup, t2: TreeCacheTup) -> int: - a, b = t1[2], t2[2] - # assert isinstance(a, str) and isinstance(b, str) - len_a, len_b = len(a), len(b) - min_len = min(len_a, len_b) - min_cmp = cmp(a[:min_len], b[:min_len]) - - if min_cmp: - return min_cmp - - return len_a - len_b - - -def merge_sort(a: List[TreeCacheTup], cmp: Callable[[TreeCacheTup, TreeCacheTup], int]) -> None: - if len(a) < 2: - return - - mid = len(a) // 2 - lefthalf = a[:mid] - righthalf = a[mid:] - - merge_sort(lefthalf, cmp) - merge_sort(righthalf, cmp) - - i = 0 - j = 0 - k = 0 - - while i < len(lefthalf) and j < len(righthalf): - if cmp(lefthalf[i], righthalf[j]) <= 0: - a[k] = lefthalf[i] - i = i + 1 - else: - a[k] = righthalf[j] - j = j + 1 - k = k + 1 - - while i < len(lefthalf): - a[k] = lefthalf[i] - i = i + 1 - k = k + 1 - - while j < len(righthalf): - a[k] = righthalf[j] - j = j + 1 - k = k + 1 +def cmp(a: str, b: str) -> int: + return (a > b) - (a < b) class TreeModifier: - """A utility class providing methods to alter the underlying cache in a list-like fashion. + """A utility class providing methods to alter the underlying cache in a list-like + fashion. - Once all adjustments are complete, the _cache, which really is a reference to - the cache of a tree, will be sorted. This ensures it will be in a serializable state. + Once all adjustments are complete, the :attr:`_cache`, which really is a reference + to the cache of a tree, will be sorted. This ensures it will be in a serializable + state. """ __slots__ = ("_cache",) @@ -126,12 +83,13 @@ def _index_by_name(self, name: str) -> int: def set_done(self) -> "TreeModifier": """Call this method once you are done modifying the tree information. - This may be called several times, but be aware that each call will cause - a sort operation. + This may be called several times, but be aware that each call will cause a sort + operation. - :return self: + :return: + self """ - merge_sort(self._cache, git_cmp) + self._cache.sort(key=lambda x: (x[2] + "/") if x[1] == Tree.tree_id << 12 else x[2]) return self # } END interface @@ -141,17 +99,21 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod """Add the given item to the tree. If an item with the given name already exists, nothing will be done, but a - ValueError will be raised if the sha and mode of the existing item do not match - the one you add, unless force is True + :exc:`ValueError` will be raised if the sha and mode of the existing item do not + match the one you add, unless `force` is ``True``. - :param sha: The 20 or 40 byte sha of the item to add + :param sha: + The 20 or 40 byte sha of the item to add. - :param mode: int representing the stat compatible mode of the item + :param mode: + :class:`int` representing the stat-compatible mode of the item. - :param force: If True, an item with your name and information will overwrite any - existing item with the same name, no matter which information it has + :param force: + If ``True``, an item with your name and information will overwrite any + existing item with the same name, no matter which information it has. - :return: self + :return: + self """ if "/" in name: raise ValueError("Name must not contain '/' characters") @@ -162,7 +124,6 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod index = self._index_by_name(name) item = (sha, mode, name) - # assert is_tree_cache(item) if index == -1: self._cache.append(item) @@ -184,7 +145,8 @@ def add_unchecked(self, binsha: bytes, mode: int, name: str) -> None: For more information on the parameters, see :meth:`add`. - :param binsha: 20 byte binary sha + :param binsha: + 20 byte binary sha. """ assert isinstance(binsha, bytes) and isinstance(mode, int) and isinstance(name, str) tree_cache = (binsha, mode, name) @@ -201,15 +163,16 @@ def __delitem__(self, name: str) -> None: class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): - """Tree objects represent an ordered list of Blobs and other Trees. + R"""Tree objects represent an ordered list of :class:`~git.objects.blob.Blob`\s and + other :class:`Tree`\s. - ``Tree as a list``:: + See :manpage:`gitglossary(7)` on "tree object": + https://git-scm.com/docs/gitglossary#def_tree_object - Access a specific blob using the - tree['filename'] notation. + Subscripting is supported, as with a list or dict: - You may as well access by index - blob = tree[0] + * Access a specific blob using the ``tree["filename"]`` notation. + * You may likewise access by index, like ``blob = tree[0]``. """ type: Literal["tree"] = "tree" @@ -225,7 +188,7 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): _map_id_to_type: Dict[int, Type[IndexObjUnion]] = { commit_id: Submodule, blob_id: Blob, - symlink_id: Blob + symlink_id: Blob, # Tree ID added once Tree is defined. } @@ -257,8 +220,8 @@ def _set_cache_(self, attr: str) -> None: # END handle attribute def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[IndexObjUnion]: - """Iterable yields tuples of (binsha, mode, name), which will be converted - to the respective object representation. + """Iterable yields tuples of (binsha, mode, name), which will be converted to + the respective object representation. """ for binsha, mode, name in iterable: path = join_path(self.path, name) @@ -268,13 +231,18 @@ def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[ raise TypeError("Unknown mode %o found in tree data for path '%s'" % (mode, path)) from e # END for each item - def join(self, file: str) -> IndexObjUnion: + def join(self, file: PathLike) -> IndexObjUnion: """Find the named object in this tree's contents. - :return: ``git.Blob`` or ``git.Tree`` or ``git.Submodule`` - :raise KeyError: if given file or tree does not exist in tree + :return: + :class:`~git.objects.blob.Blob`, :class:`Tree`, or + :class:`~git.objects.submodule.base.Submodule` + + :raise KeyError: + If the given file or tree does not exist in this tree. """ msg = "Blob or Tree named %r not found" + file = os.fspath(file) if "/" in file: tree = self item = self @@ -303,7 +271,7 @@ def join(self, file: str) -> IndexObjUnion: raise KeyError(msg % file) # END handle long paths - def __truediv__(self, file: str) -> IndexObjUnion: + def __truediv__(self, file: PathLike) -> IndexObjUnion: """The ``/`` operator is another syntax for joining. See :meth:`join` for details. @@ -312,26 +280,30 @@ def __truediv__(self, file: str) -> IndexObjUnion: @property def trees(self) -> List["Tree"]: - """:return: list(Tree, ...) list of trees directly below this tree""" + """:return: list(Tree, ...) List of trees directly below this tree""" return [i for i in self if i.type == "tree"] @property def blobs(self) -> List[Blob]: - """:return: list(Blob, ...) list of blobs directly below this tree""" + """:return: list(Blob, ...) List of blobs directly below this tree""" return [i for i in self if i.type == "blob"] @property def cache(self) -> TreeModifier: """ - :return: An object allowing to modify the internal cache. This can be used - to change the tree's contents. When done, make sure you call ``set_done`` - on the tree modifier, or serialization behaviour will be incorrect. + :return: + An object allowing modification of the internal cache. This can be used to + change the tree's contents. When done, make sure you call + :meth:`~TreeModifier.set_done` on the tree modifier, or serialization + behaviour will be incorrect. + + :note: See :class:`TreeModifier` for more information on how to alter the cache. """ return TreeModifier(self._cache) def traverse( - self, # type: ignore[override] + self, predicate: Callable[[Union[IndexObjUnion, TraversedTreeTup], int], bool] = lambda i, d: True, prune: Callable[[Union[IndexObjUnion, TraversedTreeTup], int], bool] = lambda i, d: False, depth: int = -1, @@ -340,12 +312,13 @@ def traverse( ignore_self: int = 1, as_edge: bool = False, ) -> Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]]: - """For documentation, see util.Traversable._traverse(). + """For documentation, see + `Traversable._traverse() `. - Trees are set to ``visit_once = False`` to gain more performance in the traversal. + Trees are set to ``visit_once = False`` to gain more performance in the + traversal. """ - # """ # # To typecheck instead of using cast. # import itertools # def is_tree_traversed(inp: Tuple) -> TypeGuard[Tuple[Iterator[Union['Tree', 'Blob', 'Submodule']]]]: @@ -354,13 +327,14 @@ def traverse( # ret = super().traverse(predicate, prune, depth, branch_first, visit_once, ignore_self) # ret_tup = itertools.tee(ret, 2) # assert is_tree_traversed(ret_tup), f"Type is {[type(x) for x in list(ret_tup[0])]}" - # return ret_tup[0]""" + # return ret_tup[0] + return cast( Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]], super()._traverse( - predicate, - prune, - depth, # type: ignore + predicate, # type: ignore[arg-type] + prune, # type: ignore[arg-type] + depth, branch_first, visit_once, ignore_self, @@ -369,9 +343,11 @@ def traverse( def list_traverse(self, *args: Any, **kwargs: Any) -> IterableList[IndexObjUnion]: """ - :return: IterableList with the results of the traversal as produced by - traverse() - Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']] + :return: + :class:`~git.util.IterableList` with the results of the traversal as + produced by :meth:`traverse` + + Tree -> IterableList[Union[Submodule, Tree, Blob]] """ return super()._list_traverse(*args, **kwargs) @@ -418,14 +394,15 @@ def __contains__(self, item: Union[IndexObjUnion, PathLike]) -> bool: return False def __reversed__(self) -> Iterator[IndexObjUnion]: - return reversed(self._iter_convert_to_object(self._cache)) # type: ignore + return reversed(self._iter_convert_to_object(self._cache)) # type: ignore[call-overload] def _serialize(self, stream: "BytesIO") -> "Tree": """Serialize this tree into the stream. Assumes sorted tree data. - .. note:: We will assume our tree data to be in a sorted state. If this is not - the case, serialization will not generate a correct tree representation as - these are assumed to be sorted by algorithms. + :note: + We will assume our tree data to be in a sorted state. If this is not the + case, serialization will not generate a correct tree representation as these + are assumed to be sorted by algorithms. """ tree_to_stream(self._cache, stream.write) return self diff --git a/git/objects/util.py b/git/objects/util.py index b25a3f5ff..a68d701f5 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -3,28 +3,41 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""General utility functions.""" +"""Utility functions for working with git objects.""" + +__all__ = [ + "get_object_type_by_name", + "parse_date", + "parse_actor_and_date", + "ProcessStreamAdapter", + "Traversable", + "altz_to_utctz_str", + "utctz_to_altz", + "verify_utctz", + "Actor", + "tzoffset", + "utc", +] from abc import ABC, abstractmethod import calendar from collections import deque from datetime import datetime, timedelta, tzinfo -from string import digits import re +from string import digits import time import warnings -from git.util import IterableList, IterableObj, Actor +from git.util import Actor, IterableList, IterableObj # typing ------------------------------------------------------------ + from typing import ( Any, Callable, Deque, Iterator, - # Generic, NamedTuple, - overload, Sequence, TYPE_CHECKING, Tuple, @@ -32,21 +45,23 @@ TypeVar, Union, cast, + overload, ) -from git.types import Has_id_attribute, Literal # , _T +from git.types import Has_id_attribute, Literal if TYPE_CHECKING: from io import BytesIO, StringIO - from .commit import Commit - from .blob import Blob - from .tag import TagObject - from .tree import Tree, TraversedTreeTup from subprocess import Popen - from .submodule.base import Submodule + from git.types import Protocol, runtime_checkable + + from .blob import Blob + from .commit import Commit + from .submodule.base import Submodule + from .tag import TagObject + from .tree import TraversedTreeTup, Tree else: - # Protocol = Generic[_T] # Needed for typing bug #572? Protocol = ABC def runtime_checkable(f): @@ -68,34 +83,22 @@ class TraverseNT(NamedTuple): # -------------------------------------------------------------------- -__all__ = ( - "get_object_type_by_name", - "parse_date", - "parse_actor_and_date", - "ProcessStreamAdapter", - "Traversable", - "altz_to_utctz_str", - "utctz_to_altz", - "verify_utctz", - "Actor", - "tzoffset", - "utc", -) - ZERO = timedelta(0) # { Functions def mode_str_to_int(modestr: Union[bytes, str]) -> int: - """ + """Convert mode bits from an octal mode string to an integer mode for git. + :param modestr: - String like 755 or 644 or 100644 - only the last 6 chars will be used. + String like ``755`` or ``644`` or ``100644`` - only the last 6 chars will be + used. :return: - String identifying a mode compatible to the mode methods ids of the - stat module regarding the rwx permissions for user, group and other, - special flags and file system flags, such as whether it is a symlink. + String identifying a mode compatible to the mode methods ids of the :mod:`stat` + module regarding the rwx permissions for user, group and other, special flags + and file system flags, such as whether it is a symlink. """ mode = 0 for iteration, char in enumerate(reversed(modestr[-6:])): @@ -108,13 +111,17 @@ def mode_str_to_int(modestr: Union[bytes, str]) -> int: def get_object_type_by_name( object_type_name: bytes, ) -> Union[Type["Commit"], Type["TagObject"], Type["Tree"], Type["Blob"]]: - """ - :return: A type suitable to handle the given object type name. - Use the type to create new instances. + """Retrieve the Python class GitPython uses to represent a kind of Git object. - :param object_type_name: Member of TYPES + :return: + A type suitable to handle the given as `object_type_name`. + This type can be called create new instances. - :raise ValueError: If object_type_name is unknown + :param object_type_name: + Member of :attr:`Object.TYPES `. + + :raise ValueError: + If `object_type_name` is unknown. """ if object_type_name == b"commit": from . import commit @@ -137,10 +144,11 @@ def get_object_type_by_name( def utctz_to_altz(utctz: str) -> int: - """Convert a git timezone offset into a timezone offset west of - UTC in seconds (compatible with :attr:`time.altzone`). + """Convert a git timezone offset into a timezone offset west of UTC in seconds + (compatible with :attr:`time.altzone`). - :param utctz: git utc timezone string, e.g. +0200 + :param utctz: + git utc timezone string, e.g. +0200 """ int_utctz = int(utctz) seconds = (abs(int_utctz) // 100) * 3600 + (abs(int_utctz) % 100) * 60 @@ -148,9 +156,11 @@ def utctz_to_altz(utctz: str) -> int: def altz_to_utctz_str(altz: float) -> str: - """Convert a timezone offset west of UTC in seconds into a Git timezone offset string. + """Convert a timezone offset west of UTC in seconds into a Git timezone offset + string. - :param altz: Timezone offset in seconds west of UTC + :param altz: + Timezone offset in seconds west of UTC. """ hours = abs(altz) // 3600 minutes = (abs(altz) % 3600) // 60 @@ -160,9 +170,11 @@ def altz_to_utctz_str(altz: float) -> str: def verify_utctz(offset: str) -> str: """ - :raise ValueError: If offset is incorrect + :raise ValueError: + If `offset` is incorrect. - :return: offset + :return: + `offset` """ fmt_exc = ValueError("Invalid timezone offset format: %s" % offset) if len(offset) != 5: @@ -197,7 +209,8 @@ def dst(self, dt: Union[datetime, None]) -> timedelta: def from_timestamp(timestamp: float, tz_offset: float) -> datetime: - """Convert a timestamp + tz_offset into an aware datetime instance.""" + """Convert a `timestamp` + `tz_offset` into an aware :class:`~datetime.datetime` + instance.""" utc_dt = datetime.fromtimestamp(timestamp, utc) try: local_dt = utc_dt.astimezone(tzoffset(tz_offset)) @@ -207,20 +220,21 @@ def from_timestamp(timestamp: float, tz_offset: float) -> datetime: def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: - """ - Parse the given date as one of the following: + """Parse the given date as one of the following: * Aware datetime instance * Git internal format: timestamp offset - * RFC 2822: Thu, 07 Apr 2005 22:13:13 +0200. - * ISO 8601 2005-04-07T22:13:13 - The T can be a space as well. + * :rfc:`2822`: ``Thu, 07 Apr 2005 22:13:13 +0200`` + * ISO 8601: ``2005-04-07T22:13:13`` - The ``T`` can be a space as well. - :return: Tuple(int(timestamp_UTC), int(offset)), both in seconds since epoch + :return: + Tuple(int(timestamp_UTC), int(offset)), both in seconds since epoch - :raise ValueError: If the format could not be understood + :raise ValueError: + If the format could not be understood. - :note: Date can also be YYYY.MM.DD, MM/DD/YYYY and DD.MM.YYYY. + :note: + Date can also be ``YYYY.MM.DD``, ``MM/DD/YYYY`` and ``DD.MM.YYYY``. """ if isinstance(string_date, datetime): if string_date.tzinfo: @@ -314,7 +328,8 @@ def parse_actor_and_date(line: str) -> Tuple[Actor, int, int]: author Tom Preston-Werner 1191999972 -0700 - :return: [Actor, int_seconds_since_epoch, int_timezone_offset] + :return: + [Actor, int_seconds_since_epoch, int_timezone_offset] """ actor, epoch, offset = "", "0", "0" m = _re_actor_epoch.search(line) @@ -336,8 +351,8 @@ class ProcessStreamAdapter: """Class wiring all calls to the contained Process instance. Use this type to hide the underlying process to provide access only to a specified - stream. The process is usually wrapped into an AutoInterrupt class to kill - it if the instance goes out of scope. + stream. The process is usually wrapped into an :class:`~git.cmd.Git.AutoInterrupt` + class to kill it if the instance goes out of scope. """ __slots__ = ("_proc", "_stream") @@ -352,14 +367,18 @@ def __getattr__(self, attr: str) -> Any: @runtime_checkable class Traversable(Protocol): - """Simple interface to perform depth-first or breadth-first traversals - in one direction. + """Simple interface to perform depth-first or breadth-first traversals in one + direction. Subclasses only need to implement one function. - Instances of the Subclass must be hashable. + Instances of the subclass must be hashable. + + Defined subclasses: - Defined subclasses = [Commit, Tree, SubModule] + * :class:`Commit ` + * :class:`Tree ` + * :class:`Submodule ` """ __slots__ = () @@ -368,7 +387,7 @@ class Traversable(Protocol): @abstractmethod def _get_intermediate_items(cls, item: Any) -> Sequence["Traversable"]: """ - Returns: + :return: Tuple of items connected to the given item. Must be implemented in subclass. @@ -382,7 +401,7 @@ class Tree:: (cls, Tree) -> Tuple[Tree, ...] def list_traverse(self, *args: Any, **kwargs: Any) -> Any: """Traverse self and collect all items found. - Calling this directly only the abstract base class, including via a ``super()`` + Calling this directly on the abstract base class, including via a ``super()`` proxy, is deprecated. Only overridden implementations should be called. """ warnings.warn( @@ -399,24 +418,28 @@ def _list_traverse( ) -> IterableList[Union["Commit", "Submodule", "Tree", "Blob"]]: """Traverse self and collect all items found. - :return: IterableList with the results of the traversal as produced by - :meth:`traverse`:: + :return: + :class:`~git.util.IterableList` with the results of the traversal as + produced by :meth:`traverse`:: - Commit -> IterableList['Commit'] - Submodule -> IterableList['Submodule'] - Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']] + Commit -> IterableList[Commit] + Submodule -> IterableList[Submodule] + Tree -> IterableList[Union[Submodule, Tree, Blob]] """ # Commit and Submodule have id.__attribute__ as IterableObj. # Tree has id.__attribute__ inherited from IndexObject. if isinstance(self, Has_id_attribute): id = self._id_attribute_ else: - id = "" # Shouldn't reach here, unless Traversable subclass created with no _id_attribute_. - # Could add _id_attribute_ to Traversable, or make all Traversable also Iterable? + # Shouldn't reach here, unless Traversable subclass created with no + # _id_attribute_. + id = "" + # Could add _id_attribute_ to Traversable, or make all Traversable also + # Iterable? if not as_edge: out: IterableList[Union["Commit", "Submodule", "Tree", "Blob"]] = IterableList(id) - out.extend(self.traverse(as_edge=as_edge, *args, **kwargs)) + out.extend(self.traverse(as_edge=as_edge, *args, **kwargs)) # noqa: B026 return out # Overloads in subclasses (mypy doesn't allow typing self: subclass). # Union[IterableList['Commit'], IterableList['Submodule'], IterableList[Union['Submodule', 'Tree', 'Blob']]] @@ -451,48 +474,52 @@ def _traverse( ignore_self: int = 1, as_edge: bool = False, ) -> Union[Iterator[Union["Traversable", "Blob"]], Iterator[TraversedTup]]: - """Iterator yielding items found when traversing self. + """Iterator yielding items found when traversing `self`. - :param predicate: f(i,d) returns False if item i at depth d should not be - included in the result. + :param predicate: + A function ``f(i,d)`` that returns ``False`` if item i at depth ``d`` should + not be included in the result. :param prune: - f(i,d) return True if the search should stop at item i at depth d. Item i - will not be returned. + A function ``f(i,d)`` that returns ``True`` if the search should stop at + item ``i`` at depth ``d``. Item ``i`` will not be returned. :param depth: - Defines at which level the iteration should not go deeper if -1, there is no - limit if 0, you would effectively only get self, the root of the iteration - i.e. if 1, you would only get the first level of predecessors/successors + Defines at which level the iteration should not go deeper if -1. There is no + limit if 0, you would effectively only get `self`, the root of the + iteration. If 1, you would only get the first level of + predecessors/successors. :param branch_first: - if True, items will be returned branch first, otherwise depth first + If ``True``, items will be returned branch first, otherwise depth first. :param visit_once: - if True, items will only be returned once, although they might be + If ``True``, items will only be returned once, although they might be encountered several times. Loops are prevented that way. :param ignore_self: - if True, self will be ignored and automatically pruned from the result. - Otherwise it will be the first item to be returned. If as_edge is True, the - source of the first edge is None + If ``True``, `self` will be ignored and automatically pruned from the + result. Otherwise it will be the first item to be returned. If `as_edge` is + ``True``, the source of the first edge is ``None``. :param as_edge: - if True, return a pair of items, first being the source, second the + If ``True``, return a pair of items, first being the source, second the destination, i.e. tuple(src, dest) with the edge spanning from source to - destination + destination. - :return: Iterator yielding items found when traversing self:: + :return: + Iterator yielding items found when traversing `self`:: - Commit -> Iterator[Union[Commit, Tuple[Commit, Commit]] - Submodule -> Iterator[Submodule, Tuple[Submodule, Submodule]] - Tree -> Iterator[Union[Blob, Tree, Submodule, - Tuple[Union[Submodule, Tree], Union[Blob, Tree, Submodule]]] + Commit -> Iterator[Union[Commit, Tuple[Commit, Commit]] Submodule -> + Iterator[Submodule, Tuple[Submodule, Submodule]] Tree -> + Iterator[Union[Blob, Tree, Submodule, + Tuple[Union[Submodule, Tree], Union[Blob, Tree, + Submodule]]] - ignore_self=True is_edge=True -> Iterator[item] - ignore_self=True is_edge=False --> Iterator[item] - ignore_self=False is_edge=True -> Iterator[item] | Iterator[Tuple[src, item]] - ignore_self=False is_edge=False -> Iterator[Tuple[src, item]] + ignore_self=True is_edge=True -> Iterator[item] ignore_self=True + is_edge=False --> Iterator[item] ignore_self=False is_edge=True -> + Iterator[item] | Iterator[Tuple[src, item]] ignore_self=False + is_edge=False -> Iterator[Tuple[src, item]] """ visited = set() @@ -526,7 +553,9 @@ def addToStack( visited.add(item) rval: Union[TraversedTup, "Traversable", "Blob"] - if as_edge: # If as_edge return (src, item) unless rrc is None (e.g. for first item). + if as_edge: + # If as_edge return (src, item) unless rrc is None + # (e.g. for first item). rval = (src, item) else: rval = item @@ -539,17 +568,18 @@ def addToStack( yield rval # Only continue to next level if this is appropriate! - nd = d + 1 - if depth > -1 and nd > depth: + next_d = d + 1 + if depth > -1 and next_d > depth: continue - addToStack(stack, item, branch_first, nd) + addToStack(stack, item, branch_first, next_d) # END for each item on work stack @runtime_checkable class Serializable(Protocol): - """Defines methods to serialize and deserialize objects from and into a data stream.""" + """Defines methods to serialize and deserialize objects from and into a data + stream.""" __slots__ = () @@ -557,11 +587,14 @@ class Serializable(Protocol): def _serialize(self, stream: "BytesIO") -> "Serializable": """Serialize the data of this object into the given data stream. - :note: A serialized object would :meth:`_deserialize` into the same object. + :note: + A serialized object would :meth:`_deserialize` into the same object. - :param stream: a file-like object + :param stream: + A file-like object. - :return: self + :return: + self """ raise NotImplementedError("To be implemented in subclass") @@ -569,9 +602,11 @@ def _serialize(self, stream: "BytesIO") -> "Serializable": def _deserialize(self, stream: "BytesIO") -> "Serializable": """Deserialize all information regarding this object from the stream. - :param stream: a file-like object + :param stream: + A file-like object. - :return: self + :return: + self """ raise NotImplementedError("To be implemented in subclass") @@ -584,9 +619,8 @@ class TraversableIterableObj(IterableObj, Traversable): def list_traverse(self: T_TIobj, *args: Any, **kwargs: Any) -> IterableList[T_TIobj]: return super()._list_traverse(*args, **kwargs) - @overload # type: ignore - def traverse(self: T_TIobj) -> Iterator[T_TIobj]: - ... + @overload + def traverse(self: T_TIobj) -> Iterator[T_TIobj]: ... @overload def traverse( @@ -598,8 +632,7 @@ def traverse( visit_once: bool, ignore_self: Literal[True], as_edge: Literal[False], - ) -> Iterator[T_TIobj]: - ... + ) -> Iterator[T_TIobj]: ... @overload def traverse( @@ -611,8 +644,7 @@ def traverse( visit_once: bool, ignore_self: Literal[False], as_edge: Literal[True], - ) -> Iterator[Tuple[Union[T_TIobj, None], T_TIobj]]: - ... + ) -> Iterator[Tuple[Union[T_TIobj, None], T_TIobj]]: ... @overload def traverse( @@ -624,8 +656,7 @@ def traverse( visit_once: bool, ignore_self: Literal[True], as_edge: Literal[True], - ) -> Iterator[Tuple[T_TIobj, T_TIobj]]: - ... + ) -> Iterator[Tuple[T_TIobj, T_TIobj]]: ... def traverse( self: T_TIobj, @@ -657,5 +688,13 @@ def traverse( return cast( Union[Iterator[T_TIobj], Iterator[Tuple[Union[None, T_TIobj], T_TIobj]]], - super()._traverse(predicate, prune, depth, branch_first, visit_once, ignore_self, as_edge), # type: ignore + super()._traverse( + predicate, # type: ignore[arg-type] + prune, # type: ignore[arg-type] + depth, + branch_first, + visit_once, + ignore_self, + as_edge, + ), ) diff --git a/git/refs/__init__.py b/git/refs/__init__.py index b0233e902..d6157e6f3 100644 --- a/git/refs/__init__.py +++ b/git/refs/__init__.py @@ -1,12 +1,21 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# Import all modules in order, fix the names they require. +__all__ = [ + "HEAD", + "Head", + "RefLog", + "RefLogEntry", + "Reference", + "RemoteReference", + "SymbolicReference", + "Tag", + "TagReference", +] -from .symbolic import * # noqa: F401 F403 -from .reference import * # noqa: F401 F403 -from .head import * # noqa: F401 F403 -from .tag import * # noqa: F401 F403 -from .remote import * # noqa: F401 F403 - -from .log import * # noqa: F401 F403 +from .head import HEAD, Head +from .log import RefLog, RefLogEntry +from .reference import Reference +from .remote import RemoteReference +from .symbolic import SymbolicReference +from .tag import Tag, TagReference diff --git a/git/refs/head.py b/git/refs/head.py index ebc71eb96..7f563374e 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -6,28 +6,28 @@ Note the distinction between the :class:`HEAD` and :class:`Head` classes. """ +__all__ = ["HEAD", "Head"] + from git.config import GitConfigParser, SectionConstraint -from git.util import join_path from git.exc import GitCommandError +from git.util import join_path -from .symbolic import SymbolicReference from .reference import Reference +from .symbolic import SymbolicReference # typing --------------------------------------------------- -from typing import Any, Sequence, Union, TYPE_CHECKING +from typing import Any, Sequence, TYPE_CHECKING, Union -from git.types import PathLike, Commit_ish +from git.cmd import Git +from git.types import Commit_ish, PathLike if TYPE_CHECKING: - from git.repo import Repo - from git.objects import Commit from git.refs import RemoteReference + from git.repo import Repo # ------------------------------------------------------------------- -__all__ = ["HEAD", "Head"] - def strip_quotes(string: str) -> str: if string.startswith('"') and string.endswith('"'): @@ -36,24 +36,24 @@ def strip_quotes(string: str) -> str: class HEAD(SymbolicReference): - """Special case of a SymbolicReference representing the repository's - HEAD reference.""" + """Special case of a :class:`~git.refs.symbolic.SymbolicReference` representing the + repository's HEAD reference.""" _HEAD_NAME = "HEAD" _ORIG_HEAD_NAME = "ORIG_HEAD" __slots__ = () - def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME): + def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME) -> None: if path != self._HEAD_NAME: raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path)) super().__init__(repo, path) - self.commit: "Commit" def orig_head(self) -> SymbolicReference: """ - :return: SymbolicReference pointing at the ORIG_HEAD, which is maintained - to contain the previous value of HEAD. + :return: + :class:`~git.refs.symbolic.SymbolicReference` pointing at the ORIG_HEAD, + which is maintained to contain the previous value of HEAD. """ return SymbolicReference(self.repo, self._ORIG_HEAD_NAME) @@ -63,41 +63,51 @@ def reset( index: bool = True, working_tree: bool = False, paths: Union[PathLike, Sequence[PathLike], None] = None, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> "HEAD": - """Reset our HEAD to the given commit optionally synchronizing - the index and working tree. The reference we refer to will be set to - commit as well. + """Reset our HEAD to the given commit optionally synchronizing the index and + working tree. The reference we refer to will be set to commit as well. :param commit: - Commit object, Reference Object or string identifying a revision we - should reset HEAD to. + :class:`~git.objects.commit.Commit`, :class:`~git.refs.reference.Reference`, + or string identifying a revision we should reset HEAD to. :param index: - If True, the index will be set to match the given commit. Otherwise - it will not be touched. + If ``True``, the index will be set to match the given commit. + Otherwise it will not be touched. :param working_tree: - If True, the working tree will be forcefully adjusted to match the given + If ``True``, the working tree will be forcefully adjusted to match the given commit, possibly overwriting uncommitted changes without warning. - If `working_tree` is True, `index` must be True as well. + If `working_tree` is ``True``, `index` must be ``True`` as well. :param paths: Single path or list of paths relative to the git root directory that are to be reset. This allows to partially reset individual files. + :param allow_unsafe_options: + Allow unsafe options such as ``--pathspec-from-file`` to be passed to + :manpage:`git-reset(1)`. + :param kwargs: - Additional arguments passed to git-reset. + Additional arguments passed to :manpage:`git-reset(1)`. - :return: self + :return: + self """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([commit], kwargs), + unsafe_options=Git.unsafe_git_pathspec_from_file_options, + ) mode: Union[str, None] mode = "--soft" if index: mode = "--mixed" - # Tt appears some git versions declare mixed and paths deprecated. - # See http://github.com/Byron/GitPython/issues#issue/2. + # Explicit "--mixed" when passing paths is deprecated since git 1.5.4. + # See https://github.com/gitpython-developers/GitPython/discussions/1876. if paths: mode = None # END special case @@ -123,8 +133,8 @@ def reset( class Head(Reference): - """A Head is a named reference to a Commit. Every Head instance contains a name - and a Commit object. + """A Head is a named reference to a :class:`~git.objects.commit.Commit`. Every Head + instance contains a name and a :class:`~git.objects.commit.Commit` object. Examples:: @@ -146,13 +156,12 @@ class Head(Reference): k_config_remote_ref = "merge" # Branch to merge from remote. @classmethod - def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, **kwargs: Any) -> None: + def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, **kwargs: Any) -> None: # type: ignore[override] """Delete the given heads. :param force: - If True, the heads will be deleted even if they are not yet merged into - the main development stream. - Default False + If ``True``, the heads will be deleted even if they are not yet merged into + the main development stream. Default ``False``. """ flag = "-d" if force: @@ -163,9 +172,11 @@ def set_tracking_branch(self, remote_reference: Union["RemoteReference", None]) """Configure this branch to track the given remote reference. This will alter this branch's configuration accordingly. - :param remote_reference: The remote reference to track or None to untrack - any references. - :return: self + :param remote_reference: + The remote reference to track or None to untrack any references. + + :return: + self """ from .remote import RemoteReference @@ -190,8 +201,10 @@ def set_tracking_branch(self, remote_reference: Union["RemoteReference", None]) def tracking_branch(self) -> Union["RemoteReference", None]: """ - :return: The remote_reference we are tracking, or None if we are - not a tracking branch.""" + :return: + The remote reference we are tracking, or ``None`` if we are not a tracking + branch. + """ from .remote import RemoteReference reader = self.config_reader() @@ -211,16 +224,18 @@ def rename(self, new_path: PathLike, force: bool = False) -> "Head": """Rename self to a new path. :param new_path: - Either a simple name or a path, i.e. new_name or features/new_name. - The prefix refs/heads is implied. + Either a simple name or a path, e.g. ``new_name`` or ``features/new_name``. + The prefix ``refs/heads`` is implied. :param force: - If True, the rename will succeed even if a head with the target name + If ``True``, the rename will succeed even if a head with the target name already exists. - :return: self + :return: + self - :note: Respects the ref log as git commands are used. + :note: + Respects the ref log, as git commands are used. """ flag = "-m" if force: @@ -230,7 +245,12 @@ def rename(self, new_path: PathLike, force: bool = False) -> "Head": self.path = "%s/%s" % (self._common_path_default, new_path) return self - def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]: + def checkout( + self, + force: bool = False, + allow_unsafe_options: bool = False, + **kwargs: Any, + ) -> Union["HEAD", "Head"]: """Check out this head by setting the HEAD to this reference, by updating the index to reflect the tree we point to and by updating the working tree to reflect the latest index. @@ -238,25 +258,34 @@ def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]: The command will fail if changed working tree files would be overwritten. :param force: - If True, changes to the index and the working tree will be discarded. - If False, :class:`~git.exc.GitCommandError` will be raised in that + If ``True``, changes to the index and the working tree will be discarded. + If ``False``, :exc:`~git.exc.GitCommandError` will be raised in that situation. + :param allow_unsafe_options: + Allow unsafe options such as ``--pathspec-from-file`` to be passed to + :manpage:`git-checkout(1)`. + :param kwargs: Additional keyword arguments to be passed to git checkout, e.g. ``b="new_branch"`` to create a new branch at the given spot. :return: - The active branch after the checkout operation, usually self unless - a new branch has been created. + The active branch after the checkout operation, usually self unless a new + branch has been created. If there is no active branch, as the HEAD is now detached, the HEAD reference will be returned instead. :note: - By default it is only allowed to checkout heads - everything else - will leave the HEAD detached which is allowed and possible, but remains - a special state that some tools might not be able to handle. + By default it is only allowed to checkout heads - everything else will leave + the HEAD detached which is allowed and possible, but remains a special state + that some tools might not be able to handle. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=Git.unsafe_git_pathspec_from_file_options, + ) kwargs["f"] = force if kwargs["f"] is False: kwargs.pop("f") @@ -279,15 +308,17 @@ def _config_parser(self, read_only: bool) -> SectionConstraint[GitConfigParser]: def config_reader(self) -> SectionConstraint[GitConfigParser]: """ - :return: A configuration parser instance constrained to only read - this instance's values. + :return: + A configuration parser instance constrained to only read this instance's + values. """ return self._config_parser(read_only=True) def config_writer(self) -> SectionConstraint[GitConfigParser]: """ - :return: A configuration writer instance with read-and write access - to options of this head. + :return: + A configuration writer instance with read-and write access to options of + this head. """ return self._config_parser(read_only=False) diff --git a/git/refs/log.py b/git/refs/log.py index e45798d8a..0681943bf 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -1,49 +1,48 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["RefLog", "RefLogEntry"] + from mmap import mmap import re import time as _time from git.compat import defenc from git.objects.util import ( - parse_date, Serializable, altz_to_utctz_str, + parse_date, ) from git.util import ( Actor, LockedFD, LockFile, assure_directory_exists, - to_native_path, bin_to_hex, file_contents_ro_filepath, + to_native_path, ) -import os.path as osp - - # typing ------------------------------------------------------------------ -from typing import Iterator, List, Tuple, Union, TYPE_CHECKING +from typing import Iterator, List, Tuple, TYPE_CHECKING, Union from git.types import PathLike if TYPE_CHECKING: from io import BytesIO - from git.refs import SymbolicReference + from git.config import GitConfigParser, SectionConstraint + from git.refs import SymbolicReference # ------------------------------------------------------------------------------ -__all__ = ["RefLog", "RefLogEntry"] - class RefLogEntry(Tuple[str, str, Actor, Tuple[int, int], str]): """Named tuple allowing easy access to the revlog data fields.""" _re_hexsha_only = re.compile(r"^[0-9A-Fa-f]{40}$") + __slots__ = () def __repr__(self) -> str: @@ -81,7 +80,7 @@ def actor(self) -> Actor: @property def time(self) -> Tuple[int, int]: - """time as tuple: + """Time as tuple: * [0] = ``int(time)`` * [1] = ``int(timezone_offset)`` in :attr:`time.altzone` format @@ -103,7 +102,7 @@ def new( tz_offset: int, message: str, ) -> "RefLogEntry": # skipcq: PYL-W0621 - """:return: New instance of a RefLogEntry""" + """:return: New instance of a :class:`RefLogEntry`""" if not isinstance(actor, Actor): raise ValueError("Need actor instance, got %s" % actor) # END check types @@ -111,11 +110,13 @@ def new( @classmethod def from_line(cls, line: bytes) -> "RefLogEntry": - """:return: New RefLogEntry instance from the given revlog line. + """:return: New :class:`RefLogEntry` instance from the given revlog line. - :param line: Line bytes without trailing newline + :param line: + Line bytes without trailing newline - :raise ValueError: If `line` could not be parsed + :raise ValueError: + If `line` could not be parsed. """ line_str = line.decode(defenc) fields = line_str.split("\t", 1) @@ -124,7 +125,7 @@ def from_line(cls, line: bytes) -> "RefLogEntry": elif len(fields) == 2: info, msg = fields else: - raise ValueError("Line must have up to two TAB-separated fields." " Got %s" % repr(line_str)) + raise ValueError("Line must have up to two TAB-separated fields. Got %s" % repr(line_str)) # END handle first split oldhexsha = info[:40] @@ -143,13 +144,13 @@ def from_line(cls, line: bytes) -> "RefLogEntry": actor = Actor._from_string(info[82 : email_end + 1]) time, tz_offset = parse_date(info[email_end + 2 :]) # skipcq: PYL-W0621 - return RefLogEntry((oldhexsha, newhexsha, actor, (time, tz_offset), msg)) + return RefLogEntry((oldhexsha, newhexsha, actor, (time, tz_offset), msg)) # type: ignore [arg-type] class RefLog(List[RefLogEntry], Serializable): - """A reflog contains RefLogEntrys, each of which defines a certain state - of the head in question. Custom query methods allow to retrieve log entries - by date or by other criteria. + R"""A reflog contains :class:`RefLogEntry`\s, each of which defines a certain state + of the head in question. Custom query methods allow to retrieve log entries by date + or by other criteria. Reflog entries are ordered. The first added entry is first in the list. The last entry, i.e. the last change of the head or reference, is last in the list. @@ -161,10 +162,10 @@ def __new__(cls, filepath: Union[PathLike, None] = None) -> "RefLog": inst = super().__new__(cls) return inst - def __init__(self, filepath: Union[PathLike, None] = None): + def __init__(self, filepath: Union[PathLike, None] = None) -> None: """Initialize this instance with an optional filepath, from which we will - initialize our data. The path is also used to write changes back using - the write() method.""" + initialize our data. The path is also used to write changes back using the + :meth:`write` method.""" self._path = filepath if filepath is not None: self._read_from_file() @@ -189,31 +190,43 @@ def _read_from_file(self) -> None: @classmethod def from_file(cls, filepath: PathLike) -> "RefLog": """ - :return: A new RefLog instance containing all entries from the reflog - at the given filepath - :param filepath: Path to reflog - :raise ValueError: If the file could not be read or was corrupted in some way + :return: + A new :class:`RefLog` instance containing all entries from the reflog at the + given `filepath`. + + :param filepath: + Path to reflog. + + :raise ValueError: + If the file could not be read or was corrupted in some way. """ return cls(filepath) @classmethod def path(cls, ref: "SymbolicReference") -> str: """ - :return: String to absolute path at which the reflog of the given ref - instance would be found. The path is not guaranteed to point to a valid - file though. - :param ref: SymbolicReference instance + :return: + String to absolute path at which the reflog of the given ref instance would + be found. The path is not guaranteed to point to a valid file though. + + :param ref: + :class:`~git.refs.symbolic.SymbolicReference` instance + + :raise ValueError: + If `ref.path` is invalid or escapes the repository's reflog directory. """ - return osp.join(ref.repo.git_dir, "logs", to_native_path(ref.path)) + return to_native_path(ref._get_validated_reflog_path(ref.repo, ref.path)) @classmethod def iter_entries(cls, stream: Union[str, "BytesIO", mmap]) -> Iterator[RefLogEntry]: """ - :return: Iterator yielding RefLogEntry instances, one for each line read + :return: + Iterator yielding :class:`RefLogEntry` instances, one for each line read from the given stream. - :param stream: File-like object containing the revlog in its native format - or string instance pointing to a file to read. + :param stream: + File-like object containing the revlog in its native format or string + instance pointing to a file to read. """ new_entry = RefLogEntry.from_line if isinstance(stream, str): @@ -233,24 +246,30 @@ def iter_entries(cls, stream: Union[str, "BytesIO", mmap]) -> Iterator[RefLogEnt @classmethod def entry_at(cls, filepath: PathLike, index: int) -> "RefLogEntry": """ - :return: RefLogEntry at the given index. + :return: + :class:`RefLogEntry` at the given index. - :param filepath: Full path to the index file from which to read the entry. + :param filepath: + Full path to the index file from which to read the entry. - :param index: Python list compatible index, i.e. it may be negative to - specify an entry counted from the end of the list. + :param index: + Python list compatible index, i.e. it may be negative to specify an entry + counted from the end of the list. - :raise IndexError: If the entry didn't exist. + :raise IndexError: + If the entry didn't exist. - .. note:: This method is faster as it only parses the entry at index, skipping - all other lines. Nonetheless, the whole file has to be read if - the index is negative. + :note: + This method is faster as it only parses the entry at index, skipping all + other lines. Nonetheless, the whole file has to be read if the index is + negative. """ with open(filepath, "rb") as fp: if index < 0: return RefLogEntry.from_line(fp.readlines()[index].strip()) # Read until index is reached. + line = b"" for i in range(index + 1): line = fp.readline() if not line: @@ -264,7 +283,8 @@ def entry_at(cls, filepath: PathLike, index: int) -> "RefLogEntry": def to_file(self, filepath: PathLike) -> None: """Write the contents of the reflog instance to a file at the given filepath. - :param filepath: Path to file, parent directories are assumed to exist. + :param filepath: + Path to file. Parent directories are assumed to exist. """ lfd = LockedFD(filepath) assure_directory_exists(filepath, is_file=True) @@ -290,19 +310,32 @@ def append_entry( ) -> "RefLogEntry": """Append a new log entry to the revlog at filepath. - :param config_reader: Configuration reader of the repository - used to obtain - user information. May also be an Actor instance identifying the committer - directly or None. - :param filepath: Full path to the log file. - :param oldbinsha: Binary sha of the previous commit. - :param newbinsha: Binary sha of the current commit. - :param message: Message describing the change to the reference. - :param write: If True, the changes will be written right away. Otherwise the - change will not be written. + :param config_reader: + Configuration reader of the repository - used to obtain user information. + May also be an :class:`~git.util.Actor` instance identifying the committer + directly or ``None``. + + :param filepath: + Full path to the log file. + + :param oldbinsha: + Binary sha of the previous commit. + + :param newbinsha: + Binary sha of the current commit. + + :param message: + Message describing the change to the reference. + + :param write: + If ``True``, the changes will be written right away. + Otherwise the change will not be written. - :return: RefLogEntry objects which was appended to the log. + :return: + :class:`RefLogEntry` objects which was appended to the log. - :note: As we are append-only, concurrent access is not a problem as we do not + :note: + As we are append-only, concurrent access is not a problem as we do not interfere with readers. """ @@ -340,7 +373,8 @@ def append_entry( def write(self) -> "RefLog": """Write this instance's data to the file we are originating from. - :return: self + :return: + self """ if self._path is None: raise ValueError("Instance was not initialized with a path, use to_file(...) instead") diff --git a/git/refs/reference.py b/git/refs/reference.py index 20d42472d..0c4327225 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -1,31 +1,30 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from git.util import ( - LazyMixin, - IterableObj, -) -from .symbolic import SymbolicReference, T_References +__all__ = ["Reference"] +import os +from git.util import IterableObj, LazyMixin + +from .symbolic import SymbolicReference, T_References # typing ------------------------------------------------------------------ -from typing import Any, Callable, Iterator, Type, Union, TYPE_CHECKING -from git.types import Commit_ish, PathLike, _T +from typing import Any, Callable, Iterator, TYPE_CHECKING, Type, Union + +from git.types import AnyGitObject, PathLike, _T if TYPE_CHECKING: from git.repo import Repo # ------------------------------------------------------------------------------ - -__all__ = ["Reference"] - # { Utilities def require_remote_ref_path(func: Callable[..., _T]) -> Callable[..., _T]: - """A decorator raising a TypeError if we are not a valid remote, based on the path.""" + """A decorator raising :exc:`ValueError` if we are not a valid remote, based on the + path.""" def wrapper(self: T_References, *args: Any) -> _T: if not self.is_remote(): @@ -37,7 +36,7 @@ def wrapper(self: T_References, *args: Any) -> _T: return wrapper -# }END utilities +# } END utilities class Reference(SymbolicReference, LazyMixin, IterableObj): @@ -56,13 +55,18 @@ class Reference(SymbolicReference, LazyMixin, IterableObj): def __init__(self, repo: "Repo", path: PathLike, check_path: bool = True) -> None: """Initialize this instance. - :param repo: Our parent repository. - :param path: Path relative to the .git/ directory pointing to the ref in - question, e.g. ``refs/heads/master``. - :param check_path: If False, you can provide any path. Otherwise the path must - start with the default path prefix of this type. + :param repo: + Our parent repository. + + :param path: + Path relative to the ``.git/`` directory pointing to the ref in question, + e.g. ``refs/heads/master``. + + :param check_path: + If ``False``, you can provide any path. + Otherwise the path must start with the default path prefix of this type. """ - if check_path and not str(path).startswith(self._common_path_default + "/"): + if check_path and not os.fspath(path).startswith(self._common_path_default + "/"): raise ValueError(f"Cannot instantiate {self.__class__.__name__!r} from path {path}") self.path: str # SymbolicReference converts to string at the moment. super().__init__(repo, path) @@ -75,12 +79,13 @@ def __str__(self) -> str: # @ReservedAssignment def set_object( self, - object: Union[Commit_ish, "SymbolicReference", str], + object: Union[AnyGitObject, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "Reference": """Special version which checks if the head-log needs an update as well. - :return: self + :return: + self """ oldbinsha = None if logmsg is not None: @@ -115,7 +120,10 @@ def set_object( @property def name(self) -> str: - """:return: (shortest) Name of this reference - it may contain path components""" + """ + :return: + (shortest) Name of this reference - it may contain path components + """ # The first two path tokens can be removed as they are # refs/heads or refs/tags or refs/remotes. tokens = self.path.split("/") @@ -131,34 +139,37 @@ def iter_items( *args: Any, **kwargs: Any, ) -> Iterator[T_References]: - """Equivalent to SymbolicReference.iter_items, but will return non-detached - references as well.""" + """Equivalent to + :meth:`SymbolicReference.iter_items `, + but will return non-detached references as well.""" return cls._iter_items(repo, common_path) - # }END interface + # } END interface # { Remote Interface - @property # type: ignore # mypy cannot deal with properties with an extra decorator (2021-04-21). + @property @require_remote_ref_path def remote_name(self) -> str: """ :return: - Name of the remote we are a reference of, such as 'origin' for a reference - named 'origin/master'. + Name of the remote we are a reference of, such as ``origin`` for a reference + named ``origin/master``. """ tokens = self.path.split("/") # /refs/remotes// return tokens[2] - @property # type: ignore # mypy cannot deal with properties with an extra decorator (2021-04-21). + @property @require_remote_ref_path def remote_head(self) -> str: """ - :return: Name of the remote head itself, e.g. master. + :return: + Name of the remote head itself, e.g. ``master``. - :note: The returned name is usually not qualified enough to uniquely identify - a branch. + :note: + The returned name is usually not qualified enough to uniquely identify a + branch. """ tokens = self.path.split("/") return "/".join(tokens[3:]) diff --git a/git/refs/remote.py b/git/refs/remote.py index 59d02a755..e16ae70f8 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -3,24 +3,23 @@ """Module implementing a remote object allowing easy access to git remotes.""" +__all__ = ["RemoteReference"] + import os from git.util import join_path from .head import Head - -__all__ = ["RemoteReference"] - # typing ------------------------------------------------------------------ -from typing import Any, Iterator, NoReturn, Union, TYPE_CHECKING -from git.types import PathLike +from typing import Any, Iterator, NoReturn, TYPE_CHECKING, Union +from git.types import PathLike if TYPE_CHECKING: + from git.remote import Remote from git.repo import Repo - from git import Remote # ------------------------------------------------------------------------------ @@ -47,34 +46,37 @@ def iter_items( # super is Reference return super().iter_items(repo, common_path) - # The Head implementation of delete also accepts strs, but this - # implementation does not. mypy doesn't have a way of representing - # tightening the types of arguments in subclasses and recommends Any or - # "type: ignore". (See https://github.com/python/typing/issues/241) + # The Head implementation of delete also accepts strs, but this implementation does + # not. mypy doesn't have a way of representing tightening the types of arguments in + # subclasses and recommends Any or "type: ignore". + # (See: https://github.com/python/typing/issues/241) @classmethod - def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: # type: ignore + def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: # type: ignore[override] """Delete the given remote references. :note: - kwargs are given for comparability with the base class method as we + `kwargs` are given for comparability with the base class method as we should not narrow the signature. """ + for ref in refs: + cls._check_ref_name_valid(ref.path) + repo.git.branch("-d", "-r", *refs) - # The official deletion method will ignore remote symbolic refs - these - # are generally ignored in the refs/ folder. We don't though - # and delete remainders manually. + # The official deletion method will ignore remote symbolic refs - these are + # generally ignored in the refs/ folder. We don't though and delete remainders + # manually. for ref in refs: try: - os.remove(os.path.join(repo.common_dir, ref.path)) + os.remove(cls._get_validated_path(repo.common_dir, ref.path)) except OSError: pass try: - os.remove(os.path.join(repo.git_dir, ref.path)) + os.remove(cls._get_validated_path(repo.git_dir, ref.path)) except OSError: pass # END for each ref @classmethod def create(cls, *args: Any, **kwargs: Any) -> NoReturn: - """Raises TypeError. Defined so the create method is disabled.""" + """Raise :exc:`TypeError`. Defined so the ``create`` method is disabled.""" raise TypeError("Cannot explicitly create remote references") diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 31f959ac2..824d0c46c 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -1,22 +1,25 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["SymbolicReference"] + import os +from pathlib import Path + +from gitdb.exc import BadName, BadObject from git.compat import defenc -from git.objects import Object +from git.objects.base import Object from git.objects.commit import Commit +from git.refs.log import RefLog from git.util import ( + LockedFD, + assure_directory_exists, + hex_to_bin, join_path, join_path_native, to_native_path_linux, - assure_directory_exists, - hex_to_bin, - LockedFD, ) -from gitdb.exc import BadObject, BadName - -from .log import RefLog # typing ------------------------------------------------------------------ @@ -24,21 +27,22 @@ Any, Iterator, List, + TYPE_CHECKING, Tuple, Type, TypeVar, Union, - TYPE_CHECKING, cast, ) -from git.types import Commit_ish, PathLike + +from git.types import AnyGitObject, PathLike if TYPE_CHECKING: - from git.repo import Repo - from git.refs import Head, TagReference, RemoteReference, Reference - from .log import RefLogEntry from git.config import GitConfigParser from git.objects.commit import Actor + from git.refs.log import RefLogEntry + from git.refs.reference import Reference + from git.repo import Repo T_References = TypeVar("T_References", bound="SymbolicReference") @@ -46,9 +50,6 @@ # ------------------------------------------------------------------------------ -__all__ = ["SymbolicReference"] - - def _git_dir(repo: "Repo", path: Union[PathLike, None]) -> PathLike: """Find the git dir that is appropriate for the path.""" name = f"{path}" @@ -63,7 +64,7 @@ class SymbolicReference: This does not point to a specific commit, but to another :class:`~git.refs.head.Head`, which itself specifies a commit. - A typical example for a symbolic reference is ``HEAD``. + A typical example for a symbolic reference is :class:`~git.refs.head.HEAD`. """ __slots__ = ("repo", "path") @@ -74,12 +75,12 @@ class SymbolicReference: _remote_common_path_default = "refs/remotes" _id_attribute_ = "name" - def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False): + def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False) -> None: self.repo = repo - self.path = path + self.path: PathLike = path def __str__(self) -> str: - return str(self.path) + return os.fspath(self.path) def __repr__(self) -> str: return '' % (self.__class__.__name__, self.path) @@ -100,25 +101,52 @@ def __hash__(self) -> int: def name(self) -> str: """ :return: - In case of symbolic references, the shortest assumable name - is the path itself. + In case of symbolic references, the shortest assumable name is the path + itself. """ - return str(self.path) + return os.fspath(self.path) @property def abspath(self) -> PathLike: return join_path_native(_git_dir(self.repo, self.path), self.path) + @staticmethod + def _get_validated_path(base: PathLike, path: PathLike) -> str: + path = os.fspath(path) + base_path = os.path.realpath(os.fspath(base)) + abs_path = os.path.realpath(os.path.join(base_path, path)) + try: + common_path = os.path.commonpath([base_path, abs_path]) + except ValueError as e: + raise ValueError("Reference path %r escapes the repository" % path) from e + if common_path != base_path: + raise ValueError("Reference path %r escapes the repository" % path) + return abs_path + + @classmethod + def _get_validated_ref_path(cls, repo: "Repo", path: PathLike) -> str: + """Return the absolute filesystem path for a ref after validating it.""" + cls._check_ref_name_valid(path) + ref_path = os.fspath(path) + return cls._get_validated_path(_git_dir(repo, ref_path), ref_path) + + @classmethod + def _get_validated_reflog_path(cls, repo: "Repo", path: PathLike) -> str: + """Return the absolute filesystem path for a reflog after validating it.""" + cls._check_ref_name_valid(path) + return cls._get_validated_path(os.path.join(repo.git_dir, "logs"), path) + @classmethod def _get_packed_refs_path(cls, repo: "Repo") -> str: return os.path.join(repo.common_dir, "packed-refs") @classmethod def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]: - """Return an iterator yielding pairs of sha1/path pairs (as strings) - for the corresponding refs. + """Return an iterator yielding pairs of sha1/path pairs (as strings) for the + corresponding refs. - :note: The packed refs file will be kept open as long as we iterate. + :note: + The packed refs file will be kept open as long as we iterate. """ try: with open(cls._get_packed_refs_path(repo), "rt", encoding="UTF-8") as fp: @@ -155,10 +183,12 @@ def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]: @classmethod def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> str: """ - :return: hexsha stored in the reference at the given ref_path, recursively dereferencing all - intermediate references as required + :return: + hexsha stored in the reference at the given `ref_path`, recursively + dereferencing all intermediate references as required - :param repo: The repository containing the reference at ref_path + :param repo: + The repository containing the reference at `ref_path`. """ while True: @@ -171,12 +201,11 @@ def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> def _check_ref_name_valid(ref_path: PathLike) -> None: """Check a ref name for validity. - This is based on the rules described in: - https://git-scm.com/docs/git-check-ref-format/#_description + This is based on the rules described in :manpage:`git-check-ref-format(1)`. """ previous: Union[str, None] = None one_before_previous: Union[str, None] = None - for c in str(ref_path): + for c in os.fspath(ref_path): if c in " ~^:?*[\\": raise ValueError( f"Invalid reference '{ref_path}': references cannot contain spaces, tildes (~), carets (^)," @@ -210,7 +239,7 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: raise ValueError(f"Invalid reference '{ref_path}': references cannot end with a forward slash (/)") elif previous == "@" and one_before_previous is None: raise ValueError(f"Invalid reference '{ref_path}': references cannot be '@'") - elif any(component.endswith(".lock") for component in str(ref_path).split("/")): + elif any(component.endswith(".lock") for component in Path(ref_path).parts): raise ValueError( f"Invalid reference '{ref_path}': references cannot have slash-separated components that end with" " '.lock'" @@ -221,10 +250,11 @@ def _get_ref_info_helper( cls, repo: "Repo", ref_path: Union[PathLike, None] ) -> Union[Tuple[str, None], Tuple[None, str]]: """ - :return: (str(sha), str(target_ref_path)) if available, the sha the file at - rela_path points to, or None. + :return: + *(str(sha), str(target_ref_path))*, where: - target_ref_path is the reference we point to, or None. + * *sha* is of the file at rela_path points to if available, or ``None``. + * *target_ref_path* is the reference we point to, or ``None``. """ if ref_path: cls._check_ref_name_valid(ref_path) @@ -232,7 +262,7 @@ def _get_ref_info_helper( tokens: Union[None, List[str], Tuple[str, str]] = None repodir = _git_dir(repo, ref_path) try: - with open(os.path.join(repodir, str(ref_path)), "rt", encoding="UTF-8") as fp: + with open(os.path.join(repodir, ref_path), "rt", encoding="UTF-8") as fp: # type: ignore[arg-type] value = fp.read().rstrip() # Don't only split on spaces, but on whitespace, which allows to parse lines like: # 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo @@ -266,18 +296,19 @@ def _get_ref_info_helper( @classmethod def _get_ref_info(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> Union[Tuple[str, None], Tuple[None, str]]: """ - :return: (str(sha), str(target_ref_path)) if available, the sha the file at - rela_path points to, or None. + :return: + *(str(sha), str(target_ref_path))*, where: - target_ref_path is the reference we point to, or None. + * *sha* is of the file at rela_path points to if available, or ``None``. + * *target_ref_path* is the reference we point to, or ``None``. """ return cls._get_ref_info_helper(repo, ref_path) - def _get_object(self) -> Commit_ish: + def _get_object(self) -> AnyGitObject: """ :return: - The object our ref currently refers to. Refs can be cached, they will - always point to the actual object as it gets re-created on each query. + The object our ref currently refers to. Refs can be cached, they will always + point to the actual object as it gets re-created on each query. """ # We have to be dynamic here as we may be a tag which can point to anything. # Our path will be resolved to the hexsha which will be used accordingly. @@ -286,9 +317,9 @@ def _get_object(self) -> Commit_ish: def _get_commit(self) -> "Commit": """ :return: - Commit object we point to. This works for detached and non-detached - :class:`SymbolicReference` instances. The symbolic reference will be - dereferenced recursively. + :class:`~git.objects.commit.Commit` object we point to. This works for + detached and non-detached :class:`SymbolicReference` instances. The symbolic + reference will be dereferenced recursively. """ obj = self._get_object() if obj.type == "tag": @@ -305,11 +336,15 @@ def set_commit( commit: Union[Commit, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": - """As set_object, but restricts the type of object to be a Commit. + """Like :meth:`set_object`, but restricts the type of object to be a + :class:`~git.objects.commit.Commit`. + + :raise ValueError: + If `commit` is not a :class:`~git.objects.commit.Commit` object, nor does it + point to a commit. - :raise ValueError: If commit is not a :class:`~git.objects.commit.Commit` object - or doesn't point to a commit - :return: self + :return: + self """ # Check the type - assume the best if it is a base-string. invalid_type = False @@ -336,21 +371,31 @@ def set_commit( def set_object( self, - object: Union[Commit_ish, "SymbolicReference", str], + object: Union[AnyGitObject, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": - """Set the object we point to, possibly dereference our symbolic reference first. - If the reference does not exist, it will be created. + """Set the object we point to, possibly dereference our symbolic reference + first. If the reference does not exist, it will be created. - :param object: A refspec, a :class:`SymbolicReference` or an + :param object: + A refspec, a :class:`SymbolicReference` or an :class:`~git.objects.base.Object` instance. - :class:`SymbolicReference` instances will be dereferenced beforehand to - obtain the object they point to. - :param logmsg: If not None, the message will be used in the reflog entry to be - written. Otherwise the reflog is not altered. - :note: Plain :class:`SymbolicReference` instances may not actually point to - objects by convention. - :return: self + + * :class:`SymbolicReference` instances will be dereferenced beforehand to + obtain the git object they point to. + * :class:`~git.objects.base.Object` instances must represent git objects + (:class:`~git.types.AnyGitObject`). + + :param logmsg: + If not ``None``, the message will be used in the reflog entry to be written. + Otherwise the reflog is not altered. + + :note: + Plain :class:`SymbolicReference` instances may not actually point to objects + by convention. + + :return: + self """ if isinstance(object, SymbolicReference): object = object.object # @ReservedAssignment @@ -369,45 +414,69 @@ def set_object( # set the commit on our reference return self._get_reference().set_object(object, logmsg) - commit = property(_get_commit, set_commit, doc="Query or set commits directly") # type: ignore - object = property(_get_object, set_object, doc="Return the object our ref currently refers to") # type: ignore + @property + def commit(self) -> "Commit": + """Query or set commits directly""" + return self._get_commit() + + @commit.setter + def commit(self, commit: Union[Commit, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_commit(commit) + + @property + def object(self) -> AnyGitObject: + """Return the object our ref currently refers to""" + return self._get_object() + + @object.setter + def object(self, object: Union[AnyGitObject, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_object(object) - def _get_reference(self) -> "SymbolicReference": + def _get_reference(self) -> "Reference": """ - :return: Reference Object we point to + :return: + :class:`~git.refs.reference.Reference` object we point to - :raise TypeError: If this symbolic reference is detached, hence it doesn't point - to a reference, but to a commit""" + :raise TypeError: + If this symbolic reference is detached, hence it doesn't point to a + reference, but to a commit. + """ sha, target_ref_path = self._get_ref_info(self.repo, self.path) if target_ref_path is None: raise TypeError("%s is a detached symbolic reference as it points to %r" % (self, sha)) - return self.from_path(self.repo, target_ref_path) + return cast("Reference", self.from_path(self.repo, target_ref_path)) def set_reference( self, - ref: Union[Commit_ish, "SymbolicReference", str], + ref: Union[AnyGitObject, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": - """Set ourselves to the given ref. It will stay a symbol if the ref is a Reference. - Otherwise an Object, given as Object instance or refspec, is assumed and if valid, - will be set which effectively detaches the reference if it was a purely - symbolic one. + """Set ourselves to the given `ref`. + + It will stay a symbol if the `ref` is a :class:`~git.refs.reference.Reference`. + + Otherwise a git object, specified as a :class:`~git.objects.base.Object` + instance or refspec, is assumed. If it is valid, this reference will be set to + it, which effectively detaches the reference if it was a purely symbolic one. :param ref: A :class:`SymbolicReference` instance, an :class:`~git.objects.base.Object` - instance, or a refspec string. Only if the ref is a - :class:`SymbolicReference` instance, we will point to it. Everything else is - dereferenced to obtain the actual object. + instance (specifically an :class:`~git.types.AnyGitObject`), or a refspec + string. Only if the ref is a :class:`SymbolicReference` instance, we will + point to it. Everything else is dereferenced to obtain the actual object. - :param logmsg: If set to a string, the message will be used in the reflog. + :param logmsg: + If set to a string, the message will be used in the reflog. Otherwise, a reflog entry is not written for the changed reference. The previous commit of the entry will be the commit we point to now. See also: :meth:`log_append` - :return: self + :return: + self - :note: This symbolic reference will not be dereferenced. For that, see + :note: + This symbolic reference will not be dereferenced. For that, see :meth:`set_object`. """ write_value = None @@ -442,7 +511,7 @@ def set_reference( # END handle non-existing # END retrieve old hexsha - fpath = self.abspath + fpath = self._get_validated_ref_path(self.repo, self.path) assure_directory_exists(fpath, is_file=True) lfd = LockedFD(fpath) @@ -460,18 +529,24 @@ def set_reference( return self # Aliased reference - reference: Union["Head", "TagReference", "RemoteReference", "Reference"] - reference = property(_get_reference, set_reference, doc="Returns the Reference we point to") # type: ignore + @property + def reference(self) -> "Reference": + return self._get_reference() + + @reference.setter + def reference(self, ref: Union[AnyGitObject, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_reference(ref) + ref = reference def is_valid(self) -> bool: """ :return: - True if the reference is valid, hence it can be read and points to - a valid object or reference. + ``True`` if the reference is valid, hence it can be read and points to a + valid object or reference. """ try: - self.object + self.object # noqa: B018 except (OSError, ValueError): return False else: @@ -481,22 +556,24 @@ def is_valid(self) -> bool: def is_detached(self) -> bool: """ :return: - True if we are a detached reference, hence we point to a specific commit + ``True`` if we are a detached reference, hence we point to a specific commit instead to another reference. """ try: - self.ref + self.ref # noqa: B018 return False except TypeError: return True def log(self) -> "RefLog": """ - :return: RefLog for this reference. Its last entry reflects the latest change - applied to this reference. + :return: + :class:`~git.refs.log.RefLog` for this reference. + Its last entry reflects the latest change applied to this reference. - .. note:: As the log is parsed every time, its recommended to cache it for use - instead of calling this method repeatedly. It should be considered read-only. + :note: + As the log is parsed every time, its recommended to cache it for use instead + of calling this method repeatedly. It should be considered read-only. """ return RefLog.from_file(RefLog.path(self)) @@ -508,11 +585,17 @@ def log_append( ) -> "RefLogEntry": """Append a logentry to the logfile of this ref. - :param oldbinsha: Binary sha this ref used to point to. - :param message: A message describing the change. - :param newbinsha: The sha the ref points to now. If None, our current commit sha - will be used. - :return: The added :class:`~git.refs.log.RefLogEntry` instance. + :param oldbinsha: + Binary sha this ref used to point to. + + :param message: + A message describing the change. + + :param newbinsha: + The sha the ref points to now. If None, our current commit sha will be used. + + :return: + The added :class:`~git.refs.log.RefLogEntry` instance. """ # NOTE: We use the committer of the currently active commit - this should be # correct to allow overriding the committer on a per-commit level. @@ -532,21 +615,25 @@ def log_append( def log_entry(self, index: int) -> "RefLogEntry": """ - :return: RefLogEntry at the given index + :return: + :class:`~git.refs.log.RefLogEntry` at the given index - :param index: Python list compatible positive or negative index + :param index: + Python list compatible positive or negative index. - .. note:: This method must read part of the reflog during execution, hence - it should be used sparingly, or only if you need just one index. - In that case, it will be faster than the :meth:`log` method. + :note: + This method must read part of the reflog during execution, hence it should + be used sparingly, or only if you need just one index. In that case, it will + be faster than the :meth:`log` method. """ return RefLog.entry_at(RefLog.path(self), index) @classmethod def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike: """ - :return: string with a full repository-relative path which can be used to initialize - a Reference instance, for instance by using + :return: + String with a full repository-relative path which can be used to initialize + a :class:`~git.refs.reference.Reference` instance, for instance by using :meth:`Reference.from_path `. """ if isinstance(path, SymbolicReference): @@ -554,7 +641,7 @@ def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike: full_ref_path = path if not cls._common_path_default: return full_ref_path - if not str(path).startswith(cls._common_path_default + "/"): + if not os.fspath(path).startswith(cls._common_path_default + "/"): full_ref_path = "%s/%s" % (cls._common_path_default, path) return full_ref_path @@ -566,12 +653,12 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: Repository to delete the reference from. :param path: - Short or full path pointing to the reference, e.g. ``refs/myreference`` - or just ``myreference``, hence ``refs/`` is implied. + Short or full path pointing to the reference, e.g. ``refs/myreference`` or + just ``myreference``, hence ``refs/`` is implied. Alternatively the symbolic reference to be deleted. """ full_ref_path = cls.to_full_path(path) - abs_path = os.path.join(repo.common_dir, full_ref_path) + abs_path = cls._get_validated_ref_path(repo, full_ref_path) if os.path.exists(abs_path): os.remove(abs_path) else: @@ -586,10 +673,10 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: line = line_bytes.decode(defenc) _, _, line_ref = line.partition(" ") line_ref = line_ref.strip() - # Keep line if it is a comment or if the ref to delete is not - # in the line. - # If we deleted the last line and this one is a tag-reference object, - # we drop it as well. + # Keep line if it is a comment or if the ref to delete is not in + # the line. + # If we deleted the last line and this one is a tag-reference + # object, we drop it as well. if (line.startswith("#") or full_ref_path != line_ref) and ( not dropped_last_line or dropped_last_line and not line.startswith("^") ): @@ -604,8 +691,8 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: # Write the new lines. if made_change: - # Binary writing is required, otherwise Windows will - # open the file in text mode and change LF to CRLF! + # Binary writing is required, otherwise Windows will open the file + # in text mode and change LF to CRLF! with open(pack_file_path, "wb") as fd: fd.writelines(line.encode(defenc) for line in new_lines) @@ -630,14 +717,12 @@ def _create( ) -> T_References: """Internal method used to create a new symbolic reference. - If `resolve` is False, the reference will be taken as is, creating - a proper symbolic reference. Otherwise it will be resolved to the - corresponding object and a detached symbolic reference will be created - instead. + If `resolve` is ``False``, the reference will be taken as is, creating a proper + symbolic reference. Otherwise it will be resolved to the corresponding object + and a detached symbolic reference will be created instead. """ - git_dir = _git_dir(repo, path) full_ref_path = cls.to_full_path(path) - abs_ref_path = os.path.join(git_dir, full_ref_path) + abs_ref_path = cls._get_validated_ref_path(repo, full_ref_path) # Figure out target data. target = reference @@ -647,7 +732,7 @@ def _create( if not force and os.path.isfile(abs_ref_path): target_data = str(target) if isinstance(target, SymbolicReference): - target_data = str(target.path) + target_data = os.fspath(target.path) if not resolve: target_data = "ref: " + target_data with open(abs_ref_path, "rb") as fd: @@ -679,28 +764,30 @@ def create( Repository to create the reference in. :param path: - Full path at which the new symbolic reference is supposed to be - created at, e.g. ``NEW_HEAD`` or ``symrefs/my_new_symref``. + Full path at which the new symbolic reference is supposed to be created at, + e.g. ``NEW_HEAD`` or ``symrefs/my_new_symref``. :param reference: The reference which the new symbolic reference should point to. If it is a commit-ish, the symbolic ref will be detached. :param force: - If True, force creation even if a symbolic reference with that name already exists. - Raise :class:`OSError` otherwise. + If ``True``, force creation even if a symbolic reference with that name + already exists. Raise :exc:`OSError` otherwise. :param logmsg: - If not None, the message to append to the reflog. Otherwise no reflog - entry is written. + If not ``None``, the message to append to the reflog. + If ``None``, no reflog entry is written. - :return: Newly created symbolic Reference + :return: + Newly created symbolic reference :raise OSError: - If a (Symbolic)Reference with the same name but different contents - already exists. + If a (Symbolic)Reference with the same name but different contents already + exists. - :note: This does not alter the current HEAD, index or working tree. + :note: + This does not alter the current HEAD, index or working tree. """ return cls._create(repo, path, cls._resolve_ref_on_create, reference, force, logmsg) @@ -708,24 +795,27 @@ def rename(self, new_path: PathLike, force: bool = False) -> "SymbolicReference" """Rename self to a new path. :param new_path: - Either a simple name or a full path, e.g. ``new_name`` or ``features/new_name``. + Either a simple name or a full path, e.g. ``new_name`` or + ``features/new_name``. The prefix ``refs/`` is implied for references and will be set as needed. In case this is a symbolic ref, there is no implied prefix. :param force: - If True, the rename will succeed even if a head with the target name + If ``True``, the rename will succeed even if a head with the target name already exists. It will be overwritten in that case. - :return: self + :return: + self - :raise OSError: If a file at path but with different contents already exists. + :raise OSError: + If a file at path but with different contents already exists. """ new_path = self.to_full_path(new_path) if self.path == new_path: return self - new_abs_path = os.path.join(_git_dir(self.repo, new_path), new_path) - cur_abs_path = os.path.join(_git_dir(self.repo, self.path), self.path) + new_abs_path = self._get_validated_ref_path(self.repo, new_path) + cur_abs_path = self._get_validated_ref_path(self.repo, self.path) if os.path.isfile(new_abs_path): if not force: # If they point to the same file, it's not an error. @@ -778,7 +868,7 @@ def _iter_items( # Read packed refs. for _sha, rela_path in cls._iter_packed_refs(repo): - if rela_path.startswith(str(common_path)): + if rela_path.startswith(os.fspath(common_path)): rela_paths.add(rela_path) # END relative path matches common path # END packed refs reading @@ -801,13 +891,14 @@ def iter_items( ) -> Iterator[T_References]: """Find all refs in the repository. - :param repo: is the Repo + :param repo: + The :class:`~git.repo.base.Repo`. :param common_path: Optional keyword argument to the path which is to be shared by all returned Ref objects. - Defaults to class specific portion if None, ensuring that only refs suitable - for the actual class are returned. + Defaults to class specific portion if ``None``, ensuring that only refs + suitable for the actual class are returned. :return: A list of :class:`SymbolicReference`, each guaranteed to be a symbolic ref @@ -821,13 +912,13 @@ def iter_items( @classmethod def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_References: - """ - Make a symbolic reference from a path. + """Make a symbolic reference from a path. - :param path: Full ``.git``-directory-relative path name to the Reference to - instantiate. + :param path: + Full ``.git``-directory-relative path name to the Reference to instantiate. - :note: Use :meth:`to_full_path` if you only have a partial path of a known + :note: + Use :meth:`to_full_path` if you only have a partial path of a known Reference type. :return: @@ -851,8 +942,7 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere SymbolicReference, ): try: - instance: T_References - instance = ref_type(repo, path) + instance = cast(T_References, ref_type(repo, path)) if instance.__class__ is SymbolicReference and instance.is_detached: raise ValueError("SymbolicRef was detached, we drop it") else: @@ -866,4 +956,4 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere def is_remote(self) -> bool: """:return: True if this symbolic reference points to a remote branch""" - return str(self.path).startswith(self._remote_common_path_default + "/") + return os.fspath(self.path).startswith(self._remote_common_path_default + "/") diff --git a/git/refs/tag.py b/git/refs/tag.py index a59a51337..3a7d946c1 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -1,21 +1,28 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from .reference import Reference +"""Provides a :class:`~git.refs.reference.Reference`-based type for lightweight tags. + +This defines the :class:`TagReference` class (and its alias :class:`Tag`), which +represents lightweight tags. For annotated tags (which are git objects), see the +:mod:`git.objects.tag` module. +""" __all__ = ["TagReference", "Tag"] +from .reference import Reference + # typing ------------------------------------------------------------------ -from typing import Any, Type, Union, TYPE_CHECKING -from git.types import Commit_ish, PathLike +from typing import Any, TYPE_CHECKING, Type, Union + +from git.cmd import Git +from git.types import AnyGitObject, PathLike if TYPE_CHECKING: - from git.repo import Repo - from git.objects import Commit - from git.objects import TagObject + from git.objects import Commit, TagObject from git.refs import SymbolicReference - + from git.repo import Repo # ------------------------------------------------------------------------------ @@ -36,14 +43,17 @@ class TagReference(Reference): __slots__ = () + unsafe_git_tag_options = ["--file", "-F"] + _common_default = "tags" _common_path_default = Reference._common_path_default + "/" + _common_default - @property - def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelated commit method + @property # type: ignore[misc] + def commit(self) -> "Commit": # LazyMixin has unrelated commit method """:return: Commit object the tag ref points to - :raise ValueError: If the tag points to a tree or blob + :raise ValueError: + If the tag points to a tree or blob. """ obj = self.object while obj.type != "commit": @@ -63,16 +73,18 @@ def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelat @property def tag(self) -> Union["TagObject", None]: """ - :return: Tag object this tag ref points to or None in case - we are a lightweight tag""" + :return: + Tag object this tag ref points to, or ``None`` in case we are a lightweight + tag + """ obj = self.object if obj.type == "tag": return obj return None # Make object read-only. It should be reasonably hard to adjust an existing tag. - @property - def object(self) -> Commit_ish: # type: ignore[override] + @property # type: ignore[misc] + def object(self) -> AnyGitObject: return Reference._get_object(self) @classmethod @@ -83,38 +95,55 @@ def create( reference: Union[str, "SymbolicReference"] = "HEAD", logmsg: Union[str, None] = None, force: bool = False, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> "TagReference": """Create a new tag reference. + :param repo: + The :class:`~git.repo.base.Repo` to create the tag in. + :param path: The name of the tag, e.g. ``1.0`` or ``releases/1.0``. The prefix ``refs/tags`` is implied. - :param ref: + :param reference: A reference to the :class:`~git.objects.base.Object` you want to tag. - The Object can be a commit, tree or blob. + The referenced object can be a commit, tree, or blob. :param logmsg: - If not None, the message will be used in your tag object. This will also - create an additional tag object that allows to obtain that information, e.g.:: + If not ``None``, the message will be used in your tag object. This will also + create an additional tag object that allows to obtain that information, + e.g.:: tagref.tag.message :param message: - Synonym for the `logmsg` parameter. - Included for backwards compatibility. `logmsg` takes precedence if both are passed. + Synonym for the `logmsg` parameter. Included for backwards compatibility. + `logmsg` takes precedence if both are passed. :param force: - If True, force creation of a tag even though that tag already exists. + If ``True``, force creation of a tag even though that tag already exists. + + :param allow_unsafe_options: + Allow unsafe options, such as ``--file``. :param kwargs: - Additional keyword arguments to be passed to git-tag. + Additional keyword arguments to be passed to :manpage:`git-tag(1)`. - :return: A new TagReference. + :return: + A new :class:`TagReference`. """ - if "ref" in kwargs and kwargs["ref"]: - reference = kwargs["ref"] + legacy_ref = kwargs.pop("ref", None) + if legacy_ref: + reference = legacy_ref + + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([path, reference], kwargs), + unsafe_options=cls.unsafe_git_tag_options, + clusterable_short_options="46adefilnqsv", + ) if "message" in kwargs and kwargs["message"]: kwargs["m"] = kwargs["message"] diff --git a/git/remote.py b/git/remote.py index 98a421b3a..e2d5cbc1d 100644 --- a/git/remote.py +++ b/git/remote.py @@ -5,30 +5,26 @@ """Module implementing a remote object allowing easy access to git remotes.""" +__all__ = ["RemoteProgress", "PushInfo", "FetchInfo", "Remote"] + +import contextlib import logging import re -from git.cmd import handle_process_output, Git +from git.cmd import Git, handle_process_output from git.compat import defenc, force_text +from git.config import GitConfigParser, SectionConstraint, cp from git.exc import GitCommandError +from git.refs import Head, Reference, RemoteReference, SymbolicReference, TagReference from git.util import ( - LazyMixin, - IterableObj, + CallableRemoteProgress, IterableList, + IterableObj, + LazyMixin, RemoteProgress, - CallableRemoteProgress, -) -from git.util import ( join_path, ) -from git.config import ( - GitConfigParser, - SectionConstraint, - cp, -) -from git.refs import Head, Reference, RemoteReference, SymbolicReference, TagReference - # typing------------------------------------------------------- from typing import ( @@ -47,29 +43,18 @@ overload, ) -from git.types import PathLike, Literal, Commit_ish +from git.types import AnyGitObject, Literal, PathLike if TYPE_CHECKING: - from git.repo.base import Repo + from git.objects.commit import Commit from git.objects.submodule.base import UpdateProgress - - # from git.objects.commit import Commit - # from git.objects import Blob, Tree, TagObject + from git.repo.base import Repo flagKeyLiteral = Literal[" ", "!", "+", "-", "*", "=", "t", "?"] -# def is_flagKeyLiteral(inp: str) -> TypeGuard[flagKeyLiteral]: -# return inp in [' ', '!', '+', '-', '=', '*', 't', '?'] - - # ------------------------------------------------------------- - -log = logging.getLogger("git.remote") -log.addHandler(logging.NullHandler()) - - -__all__ = ("RemoteProgress", "PushInfo", "FetchInfo", "Remote") +_logger = logging.getLogger(__name__) # { Utilities @@ -79,13 +64,15 @@ def add_progress( git: Git, progress: Union[RemoteProgress, "UpdateProgress", Callable[..., RemoteProgress], None], ) -> Any: - """Add the --progress flag to the given kwargs dict if supported by the - git command. + """Add the ``--progress`` flag to the given `kwargs` dict if supported by the git + command. - :note: If the actual progress in the given progress instance is not - given, we do not request any progress. + :note: + If the actual progress in the given progress instance is not given, we do not + request any progress. - :return: possibly altered kwargs + :return: + Possibly altered `kwargs` """ if progress is not None: v = git.version_info[:2] @@ -100,24 +87,22 @@ def add_progress( @overload -def to_progress_instance(progress: None) -> RemoteProgress: - ... +def to_progress_instance(progress: None) -> RemoteProgress: ... @overload -def to_progress_instance(progress: Callable[..., Any]) -> CallableRemoteProgress: - ... +def to_progress_instance(progress: Callable[..., Any]) -> CallableRemoteProgress: ... @overload -def to_progress_instance(progress: RemoteProgress) -> RemoteProgress: - ... +def to_progress_instance(progress: RemoteProgress) -> RemoteProgress: ... def to_progress_instance( - progress: Union[Callable[..., Any], RemoteProgress, None] + progress: Union[Callable[..., Any], RemoteProgress, None], ) -> Union[RemoteProgress, CallableRemoteProgress]: - """Given the 'progress' return a suitable object derived from RemoteProgress.""" + """Given the `progress` return a suitable object derived from + :class:`~git.util.RemoteProgress`.""" # New API only needs progress as a function. if callable(progress): return CallableRemoteProgress(progress) @@ -202,7 +187,7 @@ def __init__( self.summary = summary @property - def old_commit(self) -> Union[str, SymbolicReference, Commit_ish, None]: + def old_commit(self) -> Union["Commit", None]: return self._old_commit_sha and self._remote.repo.commit(self._old_commit_sha) or None @property @@ -228,8 +213,8 @@ def remote_ref(self) -> Union[RemoteReference, TagReference]: @classmethod def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": - """Create a new PushInfo instance as parsed from line which is expected to be like - refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes.""" + """Create a new :class:`PushInfo` instance as parsed from line which is expected + to be like refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes.""" control_character, from_to, summary = line.split("\t", 3) flags = 0 @@ -265,7 +250,7 @@ def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": flags |= cls.NEW_TAG elif "[new branch]" in summary: flags |= cls.NEW_HEAD - # uptodate encoded in control character + # `uptodate` encoded in control character else: # Fast-forward or forced update - was encoded in control character, # but we parse the old and new commit. @@ -285,7 +270,7 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> NoReturn: # -> class PushInfoList(IterableList[PushInfo]): - """IterableList of PushInfo objects.""" + """:class:`~git.util.IterableList` of :class:`PushInfo` objects.""" def __new__(cls) -> "PushInfoList": return cast(PushInfoList, IterableList.__new__(cls, "push_infos")) @@ -331,7 +316,7 @@ class FetchInfo(IterableObj): ERROR, ) = [1 << x for x in range(8)] - _re_fetch_result = re.compile(r"^\s*(.) (\[[\w\s\.$@]+\]|[\w\.$@]+)\s+(.+) -> ([^\s]+)( \(.*\)?$)?") + _re_fetch_result = re.compile(r"^ *(?:.{0,3})(.) (\[[\w \.$@]+\]|[\w\.$@]+) +(.+) -> ([^ ]+)( \(.*\)?$)?") _flag_map: Dict[flagKeyLiteral, int] = { "!": ERROR, @@ -344,19 +329,18 @@ class FetchInfo(IterableObj): @classmethod def refresh(cls) -> Literal[True]: - """This gets called by the refresh function (see the top level __init__).""" - # clear the old values in _flag_map - try: - del cls._flag_map["t"] - except KeyError: - pass + """Update information about which :manpage:`git-fetch(1)` flags are supported + by the git executable being used. - try: + Called by the :func:`git.refresh` function in the top level ``__init__``. + """ + # Clear the old values in _flag_map. + with contextlib.suppress(KeyError): + del cls._flag_map["t"] + with contextlib.suppress(KeyError): del cls._flag_map["-"] - except KeyError: - pass - # set the value given the git version + # Set the value given the git version. if Git().version_info[:2] >= (2, 10): cls._flag_map["t"] = cls.TAG_UPDATE else: @@ -369,7 +353,7 @@ def __init__( ref: SymbolicReference, flags: int, note: str = "", - old_commit: Union[Commit_ish, None] = None, + old_commit: Union[AnyGitObject, None] = None, remote_ref_path: Optional[PathLike] = None, ) -> None: """Initialize a new instance.""" @@ -388,28 +372,31 @@ def name(self) -> str: return self.ref.name @property - def commit(self) -> Commit_ish: + def commit(self) -> "Commit": """:return: Commit of our remote ref""" return self.ref.commit @classmethod def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": - """Parse information from the given line as returned by git-fetch -v - and return a new FetchInfo object representing this information. - - We can handle a line as follows: - "%c %-\\*s %-\\*s -> %s%s" - - Where c is either ' ', !, +, -, \\*, or = - ! means error - + means success forcing update - - means a tag was updated - * means birth of new branch or tag - = means the head was up to date ( and not moved ) - ' ' means a fast-forward - - fetch line is the corresponding line from FETCH_HEAD, like - acb0fa8b94ef421ad60c8507b634759a472cd56c not-for-merge branch '0.1.7RC' of /tmp/tmpya0vairemote_repo + """Parse information from the given line as returned by ``git-fetch -v`` and + return a new :class:`FetchInfo` object representing this information. + + We can handle a line as follows:: + + %c %-*s %-*s -> %s%s + + Where ``c`` is either a space, ``!``, ``+``, ``-``, ``*``, or ``=``: + + - '!' means error + - '+' means success forcing update + - '-' means a tag was updated + - '*' means birth of new branch or tag + - '=' means the head was up to date (and not moved) + - ' ' means a fast-forward + + `fetch_line` is the corresponding line from FETCH_HEAD, like:: + + acb0fa8b94ef421ad60c8507b634759a472cd56c not-for-merge branch '0.1.7RC' of /tmp/tmpya0vairemote_repo """ match = cls._re_fetch_result.match(line) if match is None: @@ -424,7 +411,6 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": remote_local_ref_str, note, ) = match.groups() - # assert is_flagKeyLiteral(control_character), f"{control_character}" control_character = cast(flagKeyLiteral, control_character) try: _new_hex_sha, _fetch_operation, fetch_note = fetch_line.split("\t") @@ -442,7 +428,7 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": # Parse operation string for more info. # This makes no sense for symbolic refs, but we parse it anyway. - old_commit: Union[Commit_ish, None] = None + old_commit: Union[AnyGitObject, None] = None is_tag_operation = False if "rejected" in operation: flags |= cls.REJECTED @@ -469,15 +455,17 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": if remote_local_ref_str == "FETCH_HEAD": ref_type = SymbolicReference elif ref_type_name == "tag" or is_tag_operation: - # The ref_type_name can be branch, whereas we are still seeing a tag operation. - # It happens during testing, which is based on actual git operations. + # The ref_type_name can be branch, whereas we are still seeing a tag + # operation. It happens during testing, which is based on actual git + # operations. ref_type = TagReference elif ref_type_name in ("remote-tracking", "branch"): - # Note: remote-tracking is just the first part of the 'remote-tracking branch' token. - # We don't parse it correctly, but its enough to know what to do, and it's new in git 1.7something. + # Note: remote-tracking is just the first part of the + # 'remote-tracking branch' token. We don't parse it correctly, but it's + # enough to know what to do, and it's new in git 1.7something. ref_type = RemoteReference elif "/" in ref_type_name: - # If the fetch spec look something like this '+refs/pull/*:refs/heads/pull/*', + # If the fetch spec look something like '+refs/pull/*:refs/heads/pull/*', # and is thus pretty much anything the user wants, we will have trouble # determining what's going on. For now, we assume the local ref is a Head. ref_type = Head @@ -489,17 +477,19 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": if ref_type is SymbolicReference: remote_local_ref = ref_type(repo, "FETCH_HEAD") else: - # Determine prefix. Tags are usually pulled into refs/tags, they may have subdirectories. - # It is not clear sometimes where exactly the item is, unless we have an absolute path as - # indicated by the 'ref/' prefix. Otherwise even a tag could be in refs/remotes, which is - # when it will have the 'tags/' subdirectory in its path. - # We don't want to test for actual existence, but try to figure everything out analytically. + # Determine prefix. Tags are usually pulled into refs/tags; they may have + # subdirectories. It is not clear sometimes where exactly the item is, + # unless we have an absolute path as indicated by the 'ref/' prefix. + # Otherwise even a tag could be in refs/remotes, which is when it will have + # the 'tags/' subdirectory in its path. We don't want to test for actual + # existence, but try to figure everything out analytically. ref_path: Optional[PathLike] = None remote_local_ref_str = remote_local_ref_str.strip() if remote_local_ref_str.startswith(Reference._common_path_default + "/"): - # Always use actual type if we get absolute paths. - # Will always be the case if something is fetched outside of refs/remotes (if its not a tag). + # Always use actual type if we get absolute paths. This will always be + # the case if something is fetched outside of refs/remotes (if its not a + # tag). ref_path = remote_local_ref_str if ref_type is not TagReference and not remote_local_ref_str.startswith( RemoteReference._common_path_default + "/" @@ -513,8 +503,8 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": ref_path = join_path(ref_type._common_path_default, remote_local_ref_str) # END obtain refpath - # Even though the path could be within the git conventions, we make - # sure we respect whatever the user wanted, and disabled path checking. + # Even though the path could be within the git conventions, we make sure we + # respect whatever the user wanted, and disabled path checking. remote_local_ref = ref_type(repo, ref_path, check_path=False) # END create ref instance @@ -527,14 +517,18 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> NoReturn: # -> raise NotImplementedError +Progress = Union[RemoteProgress, "UpdateProgress", Callable[..., RemoteProgress], None] + + class Remote(LazyMixin, IterableObj): """Provides easy read and write access to a git remote. Everything not part of this interface is considered an option for the current - remote, allowing constructs like remote.pushurl to query the pushurl. + remote, allowing constructs like ``remote.pushurl`` to query the pushurl. - :note: When querying configuration, the configuration accessor will be cached - to speed up subsequent accesses. + :note: + When querying configuration, the configuration accessor will be cached to speed + up subsequent accesses. """ __slots__ = ("repo", "name", "_config_reader") @@ -558,24 +552,29 @@ class Remote(LazyMixin, IterableObj): "--exec", ] + url: str # Obtained dynamically from _config_reader. See __getattr__ below. + """The URL configured for the remote.""" + def __init__(self, repo: "Repo", name: str) -> None: """Initialize a remote instance. - :param repo: The repository we are a remote of - :param name: The name of the remote, e.g. 'origin' + :param repo: + The repository we are a remote of. + + :param name: + The name of the remote, e.g. ``origin``. """ self.repo = repo self.name = name - self.url: str def __getattr__(self, attr: str) -> Any: - """Allows to call this instance like - remote.special( \\*args, \\*\\*kwargs) to call git-remote special self.name.""" + """Allows to call this instance like ``remote.special(*args, **kwargs)`` to + call ``git remote special self.name``.""" if attr == "_config_reader": return super().__getattr__(attr) - # Sometimes, probably due to a bug in Python itself, we are being called - # even though a slot of the same name exists. + # Sometimes, probably due to a bug in Python itself, we are being called even + # though a slot of the same name exists. try: return self._config_reader.get(attr) except cp.NoOptionError: @@ -589,7 +588,10 @@ def _set_cache_(self, attr: str) -> None: if attr == "_config_reader": # NOTE: This is cached as __getattr__ is overridden to return remote config # values implicitly, such as in print(r.pushurl). - self._config_reader = SectionConstraint(self.repo.config_reader("repository"), self._config_section_name()) + self._config_reader = SectionConstraint( + self.repo.config_reader("repository"), + self._config_section_name(), + ) else: super()._set_cache_(attr) @@ -610,7 +612,8 @@ def __hash__(self) -> int: def exists(self) -> bool: """ - :return: True if this is a valid, existing remote. + :return: + ``True`` if this is a valid, existing remote. Valid remotes have an entry in the repository's configuration. """ try: @@ -624,7 +627,7 @@ def exists(self) -> bool: @classmethod def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator["Remote"]: - """:return: Iterator yielding Remote objects of the given repository""" + """:return: Iterator yielding :class:`Remote` objects of the given repository""" for section in repo.config_reader("repository").sections(): if not section.startswith("remote "): continue @@ -638,14 +641,21 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator["Remote def set_url( self, new_url: str, old_url: Optional[str] = None, allow_unsafe_protocols: bool = False, **kwargs: Any ) -> "Remote": - """Configure URLs on current remote (cf command git remote set_url). + """Configure URLs on current remote (cf. command ``git remote set-url``). This command manages URLs on the remote. - :param new_url: String being the URL to add as an extra remote URL - :param old_url: When set, replaces this URL with new_url for the remote - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :return: self + :param new_url: + String being the URL to add as an extra remote URL. + + :param old_url: + When set, replaces this URL with `new_url` for the remote. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :return: + self """ if not allow_unsafe_protocols: Git.check_unsafe_protocols(new_url) @@ -658,25 +668,33 @@ def set_url( return self def add_url(self, url: str, allow_unsafe_protocols: bool = False, **kwargs: Any) -> "Remote": - """Adds a new url on current remote (special case of git remote set_url). + """Adds a new url on current remote (special case of ``git remote set-url``). This command adds new URLs to a given remote, making it possible to have multiple URLs for a single remote. - :param url: String being the URL to add as an extra remote URL - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :return: self + :param url: + String being the URL to add as an extra remote URL. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :return: + self """ return self.set_url(url, add=True, allow_unsafe_protocols=allow_unsafe_protocols) def delete_url(self, url: str, **kwargs: Any) -> "Remote": - """Deletes a new url on current remote (special case of git remote set_url) + """Deletes a new url on current remote (special case of ``git remote set-url``). This command deletes new URLs to a given remote, making it possible to have multiple URLs for a single remote. - :param url: String being the URL to delete from the remote - :return: self + :param url: + String being the URL to delete from the remote. + + :return: + self """ return self.set_url(url, delete=True) @@ -717,9 +735,12 @@ def urls(self) -> Iterator[str]: def refs(self) -> IterableList[RemoteReference]: """ :return: - IterableList of RemoteReference objects. It is prefixed, allowing - you to omit the remote path portion, e.g.:: - remote.refs.master # yields RemoteReference('/refs/remotes/origin/master') + :class:`~git.util.IterableList` of :class:`~git.refs.remote.RemoteReference` + objects. + + It is prefixed, allowing you to omit the remote path portion, e.g.:: + + remote.refs.master # yields RemoteReference('/refs/remotes/origin/master') """ out_refs: IterableList[RemoteReference] = IterableList(RemoteReference._id_attribute_, "%s/" % self.name) out_refs.extend(RemoteReference.list_items(self.repo, remote=self.name)) @@ -729,12 +750,13 @@ def refs(self) -> IterableList[RemoteReference]: def stale_refs(self) -> IterableList[Reference]: """ :return: - IterableList RemoteReference objects that do not have a corresponding - head in the remote reference anymore as they have been deleted on the - remote side, but are still available locally. + :class:`~git.util.IterableList` of :class:`~git.refs.remote.RemoteReference` + objects that do not have a corresponding head in the remote reference + anymore as they have been deleted on the remote side, but are still + available locally. - The IterableList is prefixed, hence the 'origin' must be omitted. See - 'refs' property for an example. + The :class:`~git.util.IterableList` is prefixed, hence the 'origin' must be + omitted. See :attr:`refs` property for an example. To make things more complicated, it can be possible for the list to include other kinds of references, for example, tag references, if these are stale @@ -763,17 +785,30 @@ def stale_refs(self) -> IterableList[Reference]: def create(cls, repo: "Repo", name: str, url: str, allow_unsafe_protocols: bool = False, **kwargs: Any) -> "Remote": """Create a new remote to the given repository. - :param repo: Repository instance that is to receive the new remote - :param name: Desired name of the remote - :param url: URL which corresponds to the remote's name - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param kwargs: Additional arguments to be passed to the git-remote add command - :return: New Remote instance - :raise GitCommandError: in case an origin with that name already exists + :param repo: + Repository instance that is to receive the new remote. + + :param name: + Desired name of the remote. + + :param url: + URL which corresponds to the remote's name. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param kwargs: + Additional arguments to be passed to the ``git remote add`` command. + + :return: + New :class:`Remote` instance + + :raise git.exc.GitCommandError: + In case an origin with that name already exists. """ scmd = "add" kwargs["insert_kwargs_after"] = scmd - url = Git.polish_url(url) + url = Git.polish_url(url, expand_vars=False) if not allow_unsafe_protocols: Git.check_unsafe_protocols(url) repo.git.remote(scmd, "--", name, url, **kwargs) @@ -788,20 +823,29 @@ def add(cls, repo: "Repo", name: str, url: str, **kwargs: Any) -> "Remote": def remove(cls, repo: "Repo", name: str) -> str: """Remove the remote with the given name. - :return: The passed remote name to remove + :return: + The passed remote name to remove """ repo.git.remote("rm", name) if isinstance(name, cls): name._clear_cache() return name - # `rm` is an alias. - rm = remove + @classmethod + def rm(cls, repo: "Repo", name: str) -> str: + """Alias of remove. + Remove the remote with the given name. + + :return: + The passed remote name to remove + """ + return cls.remove(repo, name) def rename(self, new_name: str) -> "Remote": - """Rename self to the given new_name. + """Rename self to the given `new_name`. - :return: self + :return: + self """ if self.name == new_name: return self @@ -813,12 +857,15 @@ def rename(self, new_name: str) -> "Remote": return self def update(self, **kwargs: Any) -> "Remote": - """Fetch all changes for this remote, including new branches which will - be forced in (in case your local remote branch is not part the new remote - branch's ancestry anymore). + """Fetch all changes for this remote, including new branches which will be + forced in (in case your local remote branch is not part the new remote branch's + ancestry anymore). + + :param kwargs: + Additional arguments passed to ``git remote update``. - :param kwargs: Additional arguments passed to git-remote update - :return: self + :return: + self """ scmd = "update" kwargs["insert_kwargs_after"] = scmd @@ -828,7 +875,7 @@ def update(self, **kwargs: Any) -> "Remote": def _get_fetch_info_from_stderr( self, proc: "Git.AutoInterrupt", - progress: Union[Callable[..., Any], RemoteProgress, None], + progress: Progress, kill_after_timeout: Union[None, float] = None, ) -> IterableList["FetchInfo"]: progress = to_progress_instance(progress) @@ -857,7 +904,7 @@ def _get_fetch_info_from_stderr( stderr_text = progress.error_lines and "\n".join(progress.error_lines) or "" proc.wait(stderr=stderr_text) if stderr_text: - log.warning("Error lines received while fetching: %s", stderr_text) + _logger.warning("Error lines received while fetching: %s", stderr_text) for line in progress.other_lines: line = force_text(line) @@ -878,9 +925,9 @@ def _get_fetch_info_from_stderr( msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n" msg += "Will ignore extra progress lines or fetch head lines." msg %= (l_fil, l_fhi) - log.debug(msg) - log.debug(b"info lines: " + str(fetch_info_lines).encode("UTF-8")) - log.debug(b"head info: " + str(fetch_head_info).encode("UTF-8")) + _logger.debug(msg) + _logger.debug(b"info lines: " + str(fetch_info_lines).encode("UTF-8")) + _logger.debug(b"head info: " + str(fetch_head_info).encode("UTF-8")) if l_fil < l_fhi: fetch_head_info = fetch_head_info[:l_fil] else: @@ -892,8 +939,8 @@ def _get_fetch_info_from_stderr( try: output.append(FetchInfo._from_line(self.repo, err_line, fetch_line)) except ValueError as exc: - log.debug("Caught error while parsing line: %s", exc) - log.warning("Git informed while fetching: %s", err_line.strip()) + _logger.debug("Caught error while parsing line: %s", exc) + _logger.warning("Git informed while fetching: %s", err_line.strip()) return output def _get_push_info( @@ -935,7 +982,7 @@ def stdout_handler(line: str) -> None: if not output: raise elif stderr_text: - log.warning("Error lines received while fetching: %s", stderr_text) + _logger.warning("Error lines received while fetching: %s", stderr_text) output.error = e return output @@ -956,7 +1003,7 @@ def _assert_refspec(self) -> None: def fetch( self, refspec: Union[str, List[str], None] = None, - progress: Union[RemoteProgress, None, "UpdateProgress"] = None, + progress: Progress = None, verbose: bool = True, kill_after_timeout: Union[None, float] = None, allow_unsafe_protocols: bool = False, @@ -973,31 +1020,36 @@ def fetch( "grab the master branch head from the $URL and store it as my origin branch head". And ``git push $URL refs/heads/master:refs/heads/to-upstream`` means "publish my master branch head as to-upstream branch at $URL". - See also git-push(1). + See also :manpage:`git-push(1)`. - Taken from the git manual, gitglossary(7). + Taken from the git manual, :manpage:`gitglossary(7)`. - Fetch supports multiple refspecs (as the - underlying git-fetch does) - supplying a list rather than a string - for 'refspec' will make use of this facility. + Fetch supports multiple refspecs (as the underlying :manpage:`git-fetch(1)` + does) - supplying a list rather than a string for 'refspec' will make use of + this facility. - :param progress: See :meth:`push` method. + :param progress: + See the :meth:`push` method. - :param verbose: Boolean for verbose output. + :param verbose: + Boolean for verbose output. :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process - should be killed. It is set to None by default. + should be killed. It is set to ``None`` by default. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. - :param kwargs: Additional arguments to be passed to git-fetch. + :param kwargs: + Additional arguments to be passed to :manpage:`git-fetch(1)`. :return: - IterableList(FetchInfo, ...) list of FetchInfo instances providing detailed - information about the fetch results + IterableList(FetchInfo, ...) list of :class:`FetchInfo` instances providing + detailed information about the fetch results :note: As fetch does not provide progress information to non-ttys, we cannot make @@ -1019,7 +1071,10 @@ def fetch( Git.check_unsafe_protocols(ref) if not allow_unsafe_options: - Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_fetch_options) + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_fetch_options, + ) proc = self.repo.git.fetch( "--", self, *args, as_process=True, with_stdout=False, universal_newlines=True, v=verbose, **kwargs @@ -1032,22 +1087,35 @@ def fetch( def pull( self, refspec: Union[str, List[str], None] = None, - progress: Union[RemoteProgress, "UpdateProgress", None] = None, + progress: Progress = None, kill_after_timeout: Union[None, float] = None, allow_unsafe_protocols: bool = False, allow_unsafe_options: bool = False, **kwargs: Any, ) -> IterableList[FetchInfo]: - """Pull changes from the given branch, being the same as a fetch followed - by a merge of branch with your local branch. - - :param refspec: See :meth:`fetch` method - :param progress: See :meth:`push` method - :param kill_after_timeout: See :meth:`fetch` method - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :param kwargs: Additional arguments to be passed to git-pull - :return: Please see :meth:`fetch` method + """Pull changes from the given branch, being the same as a fetch followed by a + merge of branch with your local branch. + + :param refspec: + See :meth:`fetch` method. + + :param progress: + See :meth:`push` method. + + :param kill_after_timeout: + See :meth:`fetch` method. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. + + :param kwargs: + Additional arguments to be passed to :manpage:`git-pull(1)`. + + :return: + Please see :meth:`fetch` method. """ if refspec is None: # No argument refspec, then ensure the repo's config has a fetch refspec. @@ -1060,7 +1128,10 @@ def pull( Git.check_unsafe_protocols(ref) if not allow_unsafe_options: - Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_pull_options) + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_pull_options, + ) proc = self.repo.git.pull( "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=True, v=True, **kwargs @@ -1073,7 +1144,7 @@ def pull( def push( self, refspec: Union[str, List[str], None] = None, - progress: Union[RemoteProgress, "UpdateProgress", Callable[..., RemoteProgress], None] = None, + progress: Progress = None, kill_after_timeout: Union[None, float] = None, allow_unsafe_protocols: bool = False, allow_unsafe_options: bool = False, @@ -1081,39 +1152,47 @@ def push( ) -> PushInfoList: """Push changes from source branch in refspec to target branch in refspec. - :param refspec: See :meth:`fetch` method. + :param refspec: + See :meth:`fetch` method. :param progress: Can take one of many value types: - * None to discard progress information. + * ``None``, to discard progress information. * A function (callable) that is called with the progress information. Signature: ``progress(op_code, cur_count, max_count=None, message='')``. - `Click here `__ for a description of all arguments - given to the function. - * An instance of a class derived from :class:`git.RemoteProgress` that - overrides the :meth:`~git.RemoteProgress.update` method. + See :meth:`RemoteProgress.update ` for a + description of all arguments given to the function. + * An instance of a class derived from :class:`~git.util.RemoteProgress` that + overrides the + :meth:`RemoteProgress.update ` method. - :note: No further progress information is returned after push returns. + :note: + No further progress information is returned after push returns. :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process - should be killed. It is set to None by default. + should be killed. It is set to ``None`` by default. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. :param allow_unsafe_options: - Allow unsafe options to be used, like --receive-pack. + Allow unsafe options to be used, like ``--receive-pack``. - :param kwargs: Additional arguments to be passed to git-push. + :param kwargs: + Additional arguments to be passed to :manpage:`git-push(1)`. :return: A :class:`PushInfoList` object, where each list member represents an individual head which had been updated on the remote side. + If the push contains rejected heads, these will have the - :attr:`PushInfo.ERROR` bit set in their flags. - If the operation fails completely, the length of the returned PushInfoList - will be 0. + :const:`PushInfo.ERROR` bit set in their flags. + + If the operation fails completely, the length of the returned + :class:`PushInfoList` will be 0. + Call :meth:`~PushInfoList.raise_if_error` on the returned object to raise on any failure. """ @@ -1125,7 +1204,10 @@ def push( Git.check_unsafe_protocols(ref) if not allow_unsafe_options: - Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_push_options) + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_push_options, + ) proc = self.repo.git.push( "--", @@ -1143,8 +1225,9 @@ def push( def config_reader(self) -> SectionConstraint[GitConfigParser]: """ :return: - GitConfigParser compatible object able to read options for only our remote. - Hence you may simple type config.get("pushurl") to obtain the information. + :class:`~git.config.GitConfigParser` compatible object able to read options + for only our remote. Hence you may simply type ``config.get("pushurl")`` to + obtain the information. """ return self._config_reader @@ -1158,7 +1241,9 @@ def _clear_cache(self) -> None: @property def config_writer(self) -> SectionConstraint: """ - :return: GitConfigParser compatible object able to write options for this remote. + :return: + :class:`~git.config.GitConfigParser`-compatible object able to write options + for this remote. :note: You can only own one writer at a time - delete it to release the diff --git a/git/repo/__init__.py b/git/repo/__init__.py index a63d77878..66319ef95 100644 --- a/git/repo/__init__.py +++ b/git/repo/__init__.py @@ -1,6 +1,8 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Initialize the Repo package.""" +"""Initialize the repo package.""" -from .base import Repo as Repo # noqa: F401 +__all__ = ["Repo"] + +from .base import Repo diff --git a/git/repo/base.py b/git/repo/base.py index c252ad1f3..890461959 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -5,6 +5,8 @@ from __future__ import annotations +__all__ = ["Repo"] + import gc import logging import os @@ -12,6 +14,7 @@ from pathlib import Path import re import shlex +import sys import warnings import gitdb @@ -33,29 +36,29 @@ from git.remote import Remote, add_progress, to_progress_instance from git.util import ( Actor, - finalize_process, cygpath, - hex_to_bin, expand_path, + finalize_process, + hex_to_bin, remove_password_if_present, ) from .fun import ( - rev_parse, - is_git_dir, find_submodule_git_dir, - touch, find_worktree_git_dir, + is_git_dir, + rev_parse, + touch, ) # typing ------------------------------------------------------ from git.types import ( - TBD, - PathLike, - Lit_config_levels, - Commit_ish, CallableProgress, + Commit_ish, + Lit_config_levels, + PathLike, + TBD, Tree_ish, assert_never, ) @@ -67,62 +70,68 @@ Iterator, List, Mapping, + NamedTuple, Optional, Sequence, + TYPE_CHECKING, TextIO, Tuple, Type, Union, - NamedTuple, cast, - TYPE_CHECKING, ) from git.types import ConfigLevels_Tup, TypedDict if TYPE_CHECKING: - from git.util import IterableList - from git.refs.symbolic import SymbolicReference from git.objects import Tree from git.objects.submodule.base import UpdateProgress + from git.refs.symbolic import SymbolicReference from git.remote import RemoteProgress + from git.util import IterableList # ----------------------------------------------------------- -log = logging.getLogger(__name__) - -__all__ = ("Repo",) +_logger = logging.getLogger(__name__) class BlameEntry(NamedTuple): - commit: Dict[str, "Commit"] + commit: Dict[str, Commit] linenos: range orig_path: Optional[str] orig_linenos: range class Repo: - """Represents a git repository and allows you to query references, - gather commit information, generate diffs, create and clone repositories query - the log. + """Represents a git repository and allows you to query references, create commit + information, generate diffs, create and clone repositories, and query the log. The following attributes are worth using: - 'working_dir' is the working directory of the git command, which is the working tree - directory if available or the .git directory in case of bare repositories + * :attr:`working_dir` is the working directory of the git command, which is the + working tree directory if available or the ``.git`` directory in case of bare + repositories. - 'working_tree_dir' is the working tree directory, but will return None - if we are a bare repository. + * :attr:`working_tree_dir` is the working tree directory, but will return ``None`` + if we are a bare repository. - 'git_dir' is the .git repository directory, which is always set. + * :attr:`git_dir` is the ``.git`` repository directory, which is always set. """ DAEMON_EXPORT_FILE = "git-daemon-export-ok" - git = cast("Git", None) # Must exist, or __del__ will fail in case we raise on `__init__()`. + # Must exist, or __del__ will fail in case we raise on `__init__()`. + git = cast("Git", None) + working_dir: PathLike + """The working directory of the git command.""" + + # stored as string for easier processing, but annotated as path for clearer intention _working_tree_dir: Optional[PathLike] = None + git_dir: PathLike + """The ``.git`` repository directory.""" + _common_dir: PathLike = "" # Precompiled regex @@ -133,6 +142,14 @@ class Repo: re_author_committer_start = re.compile(r"^(author|committer)") re_tab_full_line = re.compile(r"^\t(.*)$") + unsafe_git_init_options = [ + # Can install hooks that execute during later Git commands: + "--template", + # Redirects the repository metadata to a caller-controlled path: + "--separate-git-dir", + ] + """Options to :manpage:`git-init(1)` that permit unsafe code execution or I/O.""" + unsafe_git_clone_options = [ # Executes arbitrary commands: "--upload-pack", @@ -140,8 +157,14 @@ class Repo: # Can override configuration variables that execute arbitrary commands: "--config", "-c", + # Can install hooks that execute during clone: + "--template", + # Redirects the repository metadata to a caller-controlled path: + "--separate-git-dir", + # Fetches from an additional caller-controlled URI: + "--bundle-uri", ] - """Options to ``git clone`` that allow arbitrary commands to be executed. + """Options to :manpage:`git-clone(1)` that permit unsafe command execution or I/O. The ``--upload-pack``/``-u`` option allows users to execute arbitrary commands directly: @@ -150,8 +173,49 @@ class Repo: The ``--config``/``-c`` option allows users to override configuration variables like ``protocol.allow`` and ``core.gitProxy`` to execute arbitrary commands: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---configltkeygtltvaluegt + + The ``--template`` option can install hooks that execute during clone: + https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---templatetemplate-directory + + The ``--bundle-uri`` option fetches from an additional URI before fetching from the + clone URL. An untrusted value can therefore make Git access local files or + unintended network resources: + https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---bundle-uriuri """ + unsafe_git_archive_options = [ + # Allows arbitrary command execution through the remote git-upload-archive command. + "--exec", + # Writes output to a caller-controlled filesystem path. + "--output", + "-o", + # Reads from a caller-controlled filesystem path: + "--add-file", + # Injects a caller-controlled path and contents: + "--add-virtual-file", + ] + + unsafe_git_revision_options = [ + # This option allows output to be written to arbitrary files before revision parsing. + "--output", + "-o", + ] + + unsafe_git_blame_options = unsafe_git_revision_options + [ + # These options read from arbitrary files and expose their contents through blame output. + "--contents", + "-S", + "--ignore-revs-file", + ] + + unsafe_git_diff_options = unsafe_git_revision_options + [ + # Treats path operands as arbitrary filesystem paths. + "--no-index", + # Reads caller-controlled order patterns from an arbitrary file. + "-O", + "--orderfile", + ] + # Invariants config_level: ConfigLevels_Tup = ("system", "user", "global", "repository") """Represents the configuration level of a configuration file.""" @@ -168,50 +232,52 @@ def __init__( search_parent_directories: bool = False, expand_vars: bool = True, ) -> None: - """Create a new Repo instance. + R"""Create a new :class:`Repo` instance. :param path: - The path to either the root git directory or the bare git repo:: + The path to either the worktree directory or the .git directory itself:: repo = Repo("/Users/mtrier/Development/git-python") repo = Repo("/Users/mtrier/Development/git-python.git") repo = Repo("~/Development/git-python.git") repo = Repo("$REPOSITORIES/Development/git-python.git") - repo = Repo("C:\\Users\\mtrier\\Development\\git-python\\.git") + repo = Repo(R"C:\Users\mtrier\Development\git-python\.git") - - In *Cygwin*, path may be a ``cygdrive/...`` prefixed path. - - If it evaluates to false, :envvar:`GIT_DIR` is used, and if this also - evals to false, the current-directory is used. + - In *Cygwin*, `path` may be a ``cygdrive/...`` prefixed path. + - If `path` is ``None`` or an empty string, :envvar:`GIT_DIR` is used. If + that environment variable is absent or empty, the current directory is + used. :param odbt: - Object DataBase type - a type which is constructed by providing - the directory containing the database objects, i.e. .git/objects. It will - be used to access all object data + Object DataBase type - a type which is constructed by providing the + directory containing the database objects, i.e. ``.git/objects``. It will be + used to access all object data. :param search_parent_directories: - If True, all parent directories will be searched for a valid repo as well. + If ``True``, all parent directories will be searched for a valid repo as + well. Please note that this was the default behaviour in older versions of GitPython, which is considered a bug though. - :raise InvalidGitRepositoryError: - :raise NoSuchPathError: + :raise git.exc.InvalidGitRepositoryError: - :return: git.Repo + :raise git.exc.NoSuchPathError: + + :return: + :class:`Repo` """ epath = path or os.getenv("GIT_DIR") if not epath: epath = os.getcwd() + epath = os.fspath(epath) if Git.is_cygwin(): - # Given how the tests are written, this seems more likely to catch - # Cygwin git used from Windows than Windows git used from Cygwin. - # Therefore changing to Cygwin-style paths is the relevant operation. + # Given how the tests are written, this seems more likely to catch Cygwin + # git used from Windows than Windows git used from Cygwin. Therefore + # changing to Cygwin-style paths is the relevant operation. epath = cygpath(epath) - epath = epath or path or os.getcwd() - if not isinstance(epath, str): - epath = str(epath) if expand_vars and re.search(self.re_envvars, epath): warnings.warn( "The use of environment variables in paths is deprecated" @@ -223,25 +289,48 @@ def __init__( if not os.path.exists(epath): raise NoSuchPathError(epath) - ## Walk up the path to find the `.git` dir. - # + # Walk up the path to find the `.git` dir. curpath = epath git_dir = None while curpath: # ABOUT osp.NORMPATH - # It's important to normalize the paths, as submodules will otherwise initialize their - # repo instances with paths that depend on path-portions that will not exist after being - # removed. It's just cleaner. + # It's important to normalize the paths, as submodules will otherwise + # initialize their repo instances with paths that depend on path-portions + # that will not exist after being removed. It's just cleaner. + if ( + osp.isfile(osp.join(curpath, "gitdir")) + and osp.isfile(osp.join(curpath, "commondir")) + and osp.isfile(osp.join(curpath, "HEAD")) + ): + git_dir = curpath + + if "GIT_WORK_TREE" in os.environ: + self._working_tree_dir = os.getenv("GIT_WORK_TREE") + else: + # Linked worktree administrative directories store the path to the + # worktree's .git file in their gitdir file (without "gitdir: " prefix). + with open(osp.join(git_dir, "gitdir")) as fp: + worktree_gitfile = fp.read().strip() + + if not osp.isabs(worktree_gitfile): + worktree_gitfile = osp.normpath(osp.join(git_dir, worktree_gitfile)) + + self._working_tree_dir = osp.dirname(worktree_gitfile) + + break + if is_git_dir(curpath): git_dir = curpath # from man git-config : core.worktree - # Set the path to the root of the working tree. If GIT_COMMON_DIR environment - # variable is set, core.worktree is ignored and not used for determining the - # root of working tree. This can be overridden by the GIT_WORK_TREE environment - # variable. The value can be an absolute path or relative to the path to the .git - # directory, which is either specified by GIT_DIR, or automatically discovered. - # If GIT_DIR is specified but none of GIT_WORK_TREE and core.worktree is specified, - # the current working directory is regarded as the top level of your working tree. + # Set the path to the root of the working tree. If GIT_COMMON_DIR + # environment variable is set, core.worktree is ignored and not used for + # determining the root of working tree. This can be overridden by the + # GIT_WORK_TREE environment variable. The value can be an absolute path + # or relative to the path to the .git directory, which is either + # specified by GIT_DIR, or automatically discovered. If GIT_DIR is + # specified but none of GIT_WORK_TREE and core.worktree is specified, + # the current working directory is regarded as the top level of your + # working tree. self._working_tree_dir = os.path.dirname(git_dir) if os.environ.get("GIT_COMMON_DIR") is None: gitconf = self._config_reader("repository", git_dir) @@ -261,7 +350,8 @@ def __init__( sm_gitpath = find_worktree_git_dir(dotgit) if sm_gitpath is not None: - git_dir = expand_path(sm_gitpath, expand_vars) + # worktrees can use relative paths as of Git 2.48, so we join to curpath + git_dir = osp.normpath(osp.join(curpath, sm_gitpath)) self._working_tree_dir = curpath break @@ -324,10 +414,10 @@ def close(self) -> None: # they are collected by the garbage collector, thus preventing deletion. # TODO: Find these references and ensure they are closed and deleted # synchronously rather than forcing a gc collection. - if os.name == "nt": + if sys.platform == "win32": gc.collect() gitdb.util.mman.collect() - if os.name == "nt": + if sys.platform == "win32": gc.collect() def __eq__(self, rhs: object) -> bool: @@ -341,71 +431,92 @@ def __ne__(self, rhs: object) -> bool: def __hash__(self) -> int: return hash(self.git_dir) - # Description property - def _get_description(self) -> str: + @property + def description(self) -> str: + """The project's description""" filename = osp.join(self.git_dir, "description") with open(filename, "rb") as fp: return fp.read().rstrip().decode(defenc) - def _set_description(self, descr: str) -> None: + @description.setter + def description(self, descr: str) -> None: filename = osp.join(self.git_dir, "description") with open(filename, "wb") as fp: fp.write((descr + "\n").encode(defenc)) - description = property(_get_description, _set_description, doc="the project's description") - del _get_description - del _set_description - @property def working_tree_dir(self) -> Optional[PathLike]: """ - :return: The working tree directory of our git repository. - If this is a bare repository, None is returned. + :return: + The working tree directory of our git repository. + If this is a bare repository, ``None`` is returned. """ return self._working_tree_dir @property def common_dir(self) -> PathLike: """ - :return: The git dir that holds everything except possibly HEAD, - FETCH_HEAD, ORIG_HEAD, COMMIT_EDITMSG, index, and logs/. + :return: + The git dir that holds everything except possibly HEAD, FETCH_HEAD, + ORIG_HEAD, COMMIT_EDITMSG, index, and logs/. """ return self._common_dir or self.git_dir @property def bare(self) -> bool: - """:return: True if the repository is bare""" + """:return: ``True`` if the repository is bare""" return self._bare @property def heads(self) -> "IterableList[Head]": - """A list of ``Head`` objects representing the branch heads in this repo. + """A list of :class:`~git.refs.head.Head` objects representing the branch heads + in this repo. - :return: ``git.IterableList(Head, ...)`` + :return: + ``git.IterableList(Head, ...)`` """ return Head.list_items(self) + @property + def branches(self) -> "IterableList[Head]": + """Alias for heads. + A list of :class:`~git.refs.head.Head` objects representing the branch heads + in this repo. + + :return: + ``git.IterableList(Head, ...)`` + """ + return self.heads + @property def references(self) -> "IterableList[Reference]": - """A list of Reference objects representing tags, heads and remote references. + """A list of :class:`~git.refs.reference.Reference` objects representing tags, + heads and remote references. - :return: ``git.IterableList(Reference, ...)`` + :return: + ``git.IterableList(Reference, ...)`` """ return Reference.list_items(self) - # Alias for references. - refs = references + @property + def refs(self) -> "IterableList[Reference]": + """Alias for references. + A list of :class:`~git.refs.reference.Reference` objects representing tags, + heads and remote references. - # Alias for heads. - branches = heads + :return: + ``git.IterableList(Reference, ...)`` + """ + return self.references @property def index(self) -> "IndexFile": """ - :return: A :class:`~git.index.base.IndexFile` representing this repository's - index. + :return: + A :class:`~git.index.base.IndexFile` representing this repository's index. - :note: This property can be expensive, as the returned + :note: + This property can be expensive, as the returned :class:`~git.index.base.IndexFile` will be reinitialized. It is recommended to reuse the object. """ @@ -413,21 +524,27 @@ def index(self) -> "IndexFile": @property def head(self) -> "HEAD": - """:return: HEAD Object pointing to the current head reference""" + """ + :return: + :class:`~git.refs.head.HEAD` object pointing to the current head reference + """ return HEAD(self, "HEAD") @property def remotes(self) -> "IterableList[Remote]": - """A list of Remote objects allowing to access and manipulate remotes. + """A list of :class:`~git.remote.Remote` objects allowing to access and + manipulate remotes. - :return: ``git.IterableList(Remote, ...)`` + :return: + ``git.IterableList(Remote, ...)`` """ return Remote.list_items(self) def remote(self, name: str = "origin") -> "Remote": - """:return: Remote with the specified name + """:return: The remote with the specified name - :raise ValueError: If no remote with such a name exists + :raise ValueError: + If no remote with such a name exists. """ r = Remote(self, name) if not r.exists(): @@ -439,15 +556,17 @@ def remote(self, name: str = "origin") -> "Remote": @property def submodules(self) -> "IterableList[Submodule]": """ - :return: git.IterableList(Submodule, ...) of direct submodules - available from the current head + :return: + git.IterableList(Submodule, ...) of direct submodules available from the + current head """ return Submodule.list_items(self) def submodule(self, name: str) -> "Submodule": - """:return: Submodule with the given name + """:return: The submodule with the given name - :raise ValueError: If no such submodule exists + :raise ValueError: + If no such submodule exists. """ try: return self.submodules[name] @@ -458,26 +577,32 @@ def submodule(self, name: str) -> "Submodule": def create_submodule(self, *args: Any, **kwargs: Any) -> Submodule: """Create a new submodule. - :note: See the documentation of Submodule.add for a description of the - applicable parameters. + :note: + For a description of the applicable parameters, see the documentation of + :meth:`Submodule.add `. - :return: The created submodules. + :return: + The created submodule. """ return Submodule.add(self, *args, **kwargs) def iter_submodules(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: - """An iterator yielding Submodule instances, see Traversable interface - for a description of args and kwargs. + """An iterator yielding Submodule instances. - :return: Iterator + See the :class:`~git.objects.util.Traversable` interface for a description of `args` + and `kwargs`. + + :return: + Iterator """ return RootModule(self).traverse(*args, **kwargs) - def submodule_update(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: + def submodule_update(self, *args: Any, **kwargs: Any) -> RootModule: """Update the submodules, keeping the repository consistent as it will take the previous state into consideration. - :note: For more information, please see the documentation of + :note: + For more information, please see the documentation of :meth:`RootModule.update `. """ return RootModule(self).update(*args, **kwargs) @@ -486,16 +611,22 @@ def submodule_update(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: @property def tags(self) -> "IterableList[TagReference]": - """A list of ``Tag`` objects that are available in this repo. + """A list of :class:`~git.refs.tag.TagReference` objects that are available in + this repo. - :return: ``git.IterableList(TagReference, ...)`` + :return: + ``git.IterableList(TagReference, ...)`` """ return TagReference.list_items(self) def tag(self, path: PathLike) -> TagReference: - """:return: TagReference Object, reference pointing to a Commit or Tag + """ + :return: + :class:`~git.refs.tag.TagReference` object, reference pointing to a + :class:`~git.objects.commit.Commit` or tag - :param path: path to the tag reference, i.e. 0.1.5 or tags/0.1.5 + :param path: + Path to the tag reference, e.g. ``0.1.5`` or ``tags/0.1.5``. """ full_path = self._to_full_tag_path(path) return TagReference(self, full_path) @@ -519,17 +650,20 @@ def create_head( ) -> "Head": """Create a new head within the repository. - :note: For more documentation, please see the + :note: + For more documentation, please see the :meth:`Head.create ` method. - :return: Newly created :class:`~git.refs.head.Head` Reference + :return: + Newly created :class:`~git.refs.head.Head` Reference. """ return Head.create(self, path, commit, logmsg, force) def delete_head(self, *heads: "Union[str, Head]", **kwargs: Any) -> None: """Delete the given heads. - :param kwargs: Additional keyword arguments to be passed to git-branch + :param kwargs: + Additional keyword arguments to be passed to :manpage:`git-branch(1)`. """ return Head.delete(self, *heads, **kwargs) @@ -543,15 +677,17 @@ def create_tag( ) -> TagReference: """Create a new tag reference. - :note: For more documentation, please see the + :note: + For more documentation, please see the :meth:`TagReference.create ` method. - :return: :class:`~git.refs.tag.TagReference` object + :return: + :class:`~git.refs.tag.TagReference` object """ return TagReference.create(self, path, ref, message, force, **kwargs) def delete_tag(self, *tags: TagReference) -> None: - """Delete the given tag references""" + """Delete the given tag references.""" return TagReference.delete(self, *tags) def create_remote(self, name: str, url: str, **kwargs: Any) -> Remote: @@ -560,7 +696,8 @@ def create_remote(self, name: str, url: str, **kwargs: Any) -> Remote: For more information, please see the documentation of the :meth:`Remote.create ` method. - :return: :class:`~git.remote.Remote` reference + :return: + :class:`~git.remote.Remote` reference """ return Remote.create(self, name, url, **kwargs) @@ -573,7 +710,7 @@ def _get_config_path(self, config_level: Lit_config_levels, git_dir: Optional[Pa git_dir = self.git_dir # We do not support an absolute path of the gitconfig on Windows. # Use the global config instead. - if os.name == "nt" and config_level == "system": + if sys.platform == "win32" and config_level == "system": config_level = "global" if config_level == "system": @@ -590,7 +727,7 @@ def _get_config_path(self, config_level: Lit_config_levels, git_dir: Optional[Pa else: return osp.normpath(osp.join(repo_dir, "config")) else: - assert_never( # type:ignore[unreachable] + assert_never( # type: ignore[unreachable] config_level, ValueError(f"Invalid configuration level: {config_level!r}"), ) @@ -608,11 +745,12 @@ def config_reader( configuration files. :param config_level: - For possible values, see the :meth:`config_writer` method. If None, all + For possible values, see the :meth:`config_writer` method. If ``None``, all applicable levels will be used. Specify a level in case you know which file you wish to read to prevent reading multiple files. - :note: On Windows, system configuration cannot currently be read as the path is + :note: + On Windows, system configuration cannot currently be read as the path is unknown, instead the global path will be used. """ return self._config_reader(config_level=config_level) @@ -623,11 +761,7 @@ def _config_reader( git_dir: Optional[PathLike] = None, ) -> GitConfigParser: if config_level is None: - files = [ - self._get_config_path(cast(Lit_config_levels, f), git_dir) - for f in self.config_level - if cast(Lit_config_levels, f) - ] + files = [self._get_config_path(f, git_dir) for f in self.config_level if f] else: files = [self._get_config_path(config_level, git_dir)] return GitConfigParser(files, read_only=True, repo=self) @@ -650,37 +784,43 @@ def config_writer(self, config_level: Lit_config_levels = "repository") -> GitCo return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self, merge_includes=False) def commit(self, rev: Union[str, Commit_ish, None] = None) -> Commit: - """The Commit object for the specified revision. + """The :class:`~git.objects.commit.Commit` object for the specified revision. - :param rev: revision specifier, see git-rev-parse for viable options. - :return: :class:`git.Commit ` + :param rev: + Revision specifier, see :manpage:`git-rev-parse(1)` for viable options. + + :return: + :class:`~git.objects.commit.Commit` """ if rev is None: return self.head.commit return self.rev_parse(str(rev) + "^0") def iter_trees(self, *args: Any, **kwargs: Any) -> Iterator["Tree"]: - """:return: Iterator yielding Tree objects + """:return: Iterator yielding :class:`~git.objects.tree.Tree` objects - :note: Accepts all arguments known to the :meth:`iter_commits` method. + :note: + Accepts all arguments known to the :meth:`iter_commits` method. """ return (c.tree for c in self.iter_commits(*args, **kwargs)) def tree(self, rev: Union[Tree_ish, str, None] = None) -> "Tree": - """The Tree object for the given tree-ish revision. + """The :class:`~git.objects.tree.Tree` object for the given tree-ish revision. Examples:: repo.tree(repo.heads[0]) - :param rev: is a revision pointing to a Treeish (being a commit or tree) + :param rev: + A revision pointing to a Treeish (being a commit or tree). - :return: ``git.Tree`` + :return: + :class:`~git.objects.tree.Tree` :note: - If you need a non-root level tree, find it by iterating the root tree. Otherwise - it cannot know about its path relative to the repository root and subsequent - operations might have unexpected results. + If you need a non-root level tree, find it by iterating the root tree. + Otherwise it cannot know about its path relative to the repository root and + subsequent operations might have unexpected results. """ if rev is None: return self.head.commit.tree @@ -690,56 +830,84 @@ def iter_commits( self, rev: Union[str, Commit, "SymbolicReference", None] = None, paths: Union[PathLike, Sequence[PathLike]] = "", + allow_unsafe_options: bool = False, **kwargs: Any, ) -> Iterator[Commit]: - """A list of Commit objects representing the history of a given ref/commit. + """An iterator of :class:`~git.objects.commit.Commit` objects representing the + history of a given ref/commit. :param rev: - Revision specifier, see git-rev-parse for viable options. - If None, the active branch will be used. + Revision specifier, see :manpage:`git-rev-parse(1)` for viable options. + If ``None``, the active branch will be used. :param paths: - An optional path or a list of paths; if set only commits that include the - path or paths will be returned + An optional path or a list of paths. If set, only commits that include the + path or paths will be returned. :param kwargs: - Arguments to be passed to git-rev-list - common ones are max_count and skip. + Arguments to be passed to :manpage:`git-rev-list(1)`. + Common ones are ``max_count`` and ``skip``. + + :param allow_unsafe_options: + Allow unsafe options in the revision argument, like ``--output``. - :note: To receive only commits between two named revisions, use the + :note: + To receive only commits between two named revisions, use the ``"revA...revB"`` revision specifier. - :return: ``git.Commit[]`` + :return: + Iterator of :class:`~git.objects.commit.Commit` objects """ if rev is None: rev = self.head.commit - return Commit.iter_items(self, rev, paths, **kwargs) + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([rev], kwargs), unsafe_options=self.unsafe_git_revision_options + ) + + return Commit.iter_items( + self, + rev, + paths, + allow_unsafe_options=allow_unsafe_options, + **kwargs, + ) + + def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Commit]: + R"""Find the closest common ancestor for the given revision + (:class:`~git.objects.commit.Commit`\s, :class:`~git.refs.tag.Tag`\s, + :class:`~git.refs.reference.Reference`\s, etc.). + + :param rev: + At least two revs to find the common ancestor for. - def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: - """Find the closest common ancestor for the given revision (Commits, Tags, References, etc.). + :param kwargs: + Additional arguments to be passed to the ``repo.git.merge_base()`` command + which does all the work. - :param rev: At least two revs to find the common ancestor for. - :param kwargs: Additional arguments to be passed to the - ``repo.git.merge_base()`` command which does all the work. - :return: A list of :class:`~git.objects.commit.Commit` objects. If ``--all`` was + :return: + A list of :class:`~git.objects.commit.Commit` objects. If ``--all`` was not passed as a keyword argument, the list will have at max one :class:`~git.objects.commit.Commit`, or is empty if no common merge base exists. - :raises ValueError: If not at least two revs are provided. + + :raise ValueError: + If fewer than two revisions are provided. """ if len(rev) < 2: raise ValueError("Please specify at least two revs, got only %i" % len(rev)) # END handle input - res: List[Union[Commit_ish, None]] = [] + res: List[Commit] = [] try: - lines = self.git.merge_base(*rev, **kwargs).splitlines() # List[str] + lines: List[str] = self.git.merge_base(*rev, **kwargs).splitlines() except GitCommandError as err: if err.status == 128: raise # END handle invalid rev - # Status code 1 is returned if there is no merge-base - # (see https://github.com/git/git/blob/master/builtin/merge-base.c#L16) + # Status code 1 is returned if there is no merge-base. + # (See: https://github.com/git/git/blob/v2.44.0/builtin/merge-base.c#L19) return res # END exception handling @@ -749,12 +917,17 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: return res - def is_ancestor(self, ancestor_rev: "Commit", rev: "Commit") -> bool: + def is_ancestor(self, ancestor_rev: Commit, rev: Commit) -> bool: """Check if a commit is an ancestor of another. - :param ancestor_rev: Rev which should be an ancestor - :param rev: Rev to test against ancestor_rev - :return: ``True``, ancestor_rev is an ancestor to rev. + :param ancestor_rev: + Rev which should be an ancestor. + + :param rev: + Rev to test against `ancestor_rev`. + + :return: + ``True`` if `ancestor_rev` is an ancestor to `rev`. """ try: self.git.merge_base(ancestor_rev, rev, is_ancestor=True) @@ -772,7 +945,7 @@ def is_valid_object(self, sha: str, object_type: Union[str, None] = None) -> boo if object_info.type == object_type.encode(): return True else: - log.debug( + _logger.debug( "Commit hash points to an object of type '%s'. Requested were objects of type '%s'", object_info.type.decode(), object_type, @@ -781,38 +954,43 @@ def is_valid_object(self, sha: str, object_type: Union[str, None] = None) -> boo else: return True except BadObject: - log.debug("Commit hash is invalid.") + _logger.debug("Commit hash is invalid.") return False def _get_daemon_export(self) -> bool: - if self.git_dir: - filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) + git_dir = getattr(self, "git_dir", None) + if git_dir is None: + return False + filename = osp.join(git_dir, self.DAEMON_EXPORT_FILE) return osp.exists(filename) def _set_daemon_export(self, value: object) -> None: - if self.git_dir: - filename = osp.join(self.git_dir, self.DAEMON_EXPORT_FILE) + git_dir = getattr(self, "git_dir", None) + if git_dir is None: + return + filename = osp.join(git_dir, self.DAEMON_EXPORT_FILE) fileexists = osp.exists(filename) if value and not fileexists: touch(filename) elif not value and fileexists: os.unlink(filename) - daemon_export = property( - _get_daemon_export, - _set_daemon_export, - doc="If True, git-daemon may export this repository", - ) - del _get_daemon_export - del _set_daemon_export + @property + def daemon_export(self) -> bool: + """If True, git-daemon may export this repository""" + return self._get_daemon_export() + + @daemon_export.setter + def daemon_export(self, value: object) -> None: + self._set_daemon_export(value) def _get_alternates(self) -> List[str]: """The list of alternates for this repo from which objects can be retrieved. - :return: List of strings being pathnames of alternates + :return: + List of strings being pathnames of alternates """ - if self.git_dir: - alternates_path = osp.join(self.git_dir, "objects", "info", "alternates") + alternates_path = osp.join(self.common_dir, "objects", "info", "alternates") if osp.exists(alternates_path): with open(alternates_path, "rb") as f: @@ -821,17 +999,17 @@ def _get_alternates(self) -> List[str]: return [] def _set_alternates(self, alts: List[str]) -> None: - """Sets the alternates. + """Set the alternates. :param alts: - is the array of string paths representing the alternates at which - git should look for objects, i.e. /home/user/repo/.git/objects + The array of string paths representing the alternates at which git should + look for objects, i.e. ``/home/user/repo/.git/objects``. - :raise NoSuchPathError: + :raise git.exc.NoSuchPathError: :note: - The method does not check for the existence of the paths in alts - as the caller is responsible. + The method does not check for the existence of the paths in `alts`, as the + caller is responsible. """ alternates_path = osp.join(self.common_dir, "objects", "info", "alternates") if not alts: @@ -841,11 +1019,14 @@ def _set_alternates(self, alts: List[str]) -> None: with open(alternates_path, "wb") as f: f.write("\n".join(alts).encode(defenc)) - alternates = property( - _get_alternates, - _set_alternates, - doc="Retrieve a list of alternates paths or set a list paths to be used as alternates", - ) + @property + def alternates(self) -> List[str]: + """Retrieve a list of alternates paths or set a list paths to be used as alternates""" + return self._get_alternates() + + @alternates.setter + def alternates(self, alts: List[str]) -> None: + self._set_alternates(alts) def is_dirty( self, @@ -858,8 +1039,8 @@ def is_dirty( """ :return: ``True`` if the repository is considered dirty. By default it will react - like a git-status without untracked files, hence it is dirty if the - index or the working copy have changes. + like a :manpage:`git-status(1)` without untracked files, hence it is dirty + if the index or the working copy have changes. """ if self._bare: # Bare repositories with no associated working directory are @@ -871,7 +1052,7 @@ def is_dirty( if not submodules: default_args.append("--ignore-submodules") if path: - default_args.extend(["--", str(path)]) + default_args.extend(["--", os.fspath(path)]) if index: # diff index against HEAD. if osp.isfile(self.index.path) and len(self.git.diff("--cached", *default_args)): @@ -894,11 +1075,12 @@ def untracked_files(self) -> List[str]: :return: list(str,...) - Files currently untracked as they have not been staged yet. Paths - are relative to the current working directory of the git command. + Files currently untracked as they have not been staged yet. Paths are + relative to the current working directory of the git command. :note: Ignored files will not appear here, i.e. files mentioned in ``.gitignore``. + :note: This property is expensive, as no cache is involved. To process the result, please consider caching it yourself. @@ -926,23 +1108,25 @@ def _get_untracked_files(self, *args: Any, **kwargs: Any) -> List[str]: return untracked_files def ignored(self, *paths: PathLike) -> List[str]: - """Checks if paths are ignored via .gitignore. + """Checks if paths are ignored via ``.gitignore``. - This does so using the ``git check-ignore`` method. + This does so using the :manpage:`git-check-ignore(1)` method. - :param paths: List of paths to check whether they are ignored or not + :param paths: + List of paths to check whether they are ignored or not. - :return: Subset of those paths which are ignored + :return: + Subset of those paths which are ignored """ try: proc: str = self.git.check_ignore(*paths) except GitCommandError as err: - # If return code is 1, this means none of the items in *paths - # are ignored by Git, so return an empty list. Raise the - # exception on all other return codes. if err.status == 1: + # If return code is 1, this means none of the items in *paths are + # ignored by Git, so return an empty list. return [] else: + # Raise the exception on all other return codes. raise return proc.replace("\\\\", "\\").replace('"', "").split("\n") @@ -951,27 +1135,53 @@ def ignored(self, *paths: PathLike) -> List[str]: def active_branch(self) -> Head: """The name of the currently active branch. - :raises TypeError: If HEAD is detached - :return: Head to the active branch + :raise TypeError: + If HEAD is detached. + + :raise ValueError: + If HEAD points to the ``.invalid`` ref Git uses to mark refs as + incompatible with older clients. + + :return: + :class:`~git.refs.head.Head` to the active branch """ - # reveal_type(self.head.reference) # => Reference - return self.head.reference + active_branch = self.head.reference + if active_branch.name == ".invalid": + raise ValueError( + "HEAD points to 'refs/heads/.invalid', which Git uses to mark refs as incompatible with older clients" + ) + return active_branch - def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterator["BlameEntry"]: + def blame_incremental( + self, rev: str | HEAD | None, file: str, allow_unsafe_options: bool = False, **kwargs: Any + ) -> Iterator["BlameEntry"]: """Iterator for blame information for the given file at the given revision. Unlike :meth:`blame`, this does not return the actual file's contents, only a stream of :class:`BlameEntry` tuples. - :param rev: Revision specifier, see git-rev-parse for viable options. + :param rev: + Revision specifier. If ``None``, the blame will include all the latest + uncommitted changes. Otherwise, anything successfully parsed by + :manpage:`git-rev-parse(1)` is a valid option. - :return: Lazy iterator of :class:`BlameEntry` tuples, where the commit indicates - the commit to blame for the line, and range indicates a span of line numbers - in the resulting file. + :param allow_unsafe_options: + Allow unsafe options in revision argument, like ``--output`` or ``--contents``. + + :return: + Lazy iterator of :class:`BlameEntry` tuples, where the commit indicates the + commit to blame for the line, and range indicates a span of line numbers in + the resulting file. If you combine all line number ranges outputted by this command, you should get a continuous range spanning all line numbers in the file. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([rev], kwargs), + unsafe_options=self.unsafe_git_blame_options, + clusterable_short_options="46bceflnpqstvw", + ) data: bytes = self.git.blame(rev, "--", file, p=True, incremental=True, stdout_as_string=False, **kwargs) commits: Dict[bytes, Commit] = {} @@ -979,7 +1189,8 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat stream = (line for line in data.split(b"\n") if line) while True: try: - line = next(stream) # When exhausted, causes a StopIteration, terminating this function. + # When exhausted, causes a StopIteration, terminating this function. + line = next(stream) except StopIteration: return split_line = line.split() @@ -988,8 +1199,8 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat num_lines = int(num_lines_b) orig_lineno = int(orig_lineno_b) if hexsha not in commits: - # Now read the next few lines and build up a dict of properties - # for this commit. + # Now read the next few lines and build up a dict of properties for this + # commit. props: Dict[bytes, bytes] = {} while True: try: @@ -997,8 +1208,8 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat except StopIteration: return if line == b"boundary": - # "boundary" indicates a root commit and occurs - # instead of the "previous" tag. + # "boundary" indicates a root commit and occurs instead of the + # "previous" tag. continue tag, value = line.split(b" ", 1) @@ -1024,11 +1235,12 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat ) commits[hexsha] = c else: - # Discard all lines until we find "filename" which is - # guaranteed to be the last line. + # Discard all lines until we find "filename" which is guaranteed to be + # the last line. while True: try: - line = next(stream) # Will fail if we reach the EOF unexpectedly. + # Will fail if we reach the EOF unexpectedly. + line = next(stream) except StopIteration: return tag, value = line.split(b" ", 1) @@ -1045,27 +1257,41 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat def blame( self, - rev: Union[str, HEAD], + rev: Union[str, HEAD, None], file: str, incremental: bool = False, - rev_opts: Optional[List[str]] = None, + rev_opts: Optional[Sequence[str]] = None, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> List[List[Commit | List[str | bytes] | None]] | Iterator[BlameEntry] | None: """The blame information for the given file at the given revision. - :param rev: Revision specifier, see git-rev-parse for viable options. + :param rev: + Revision specifier. If ``None``, the blame will include all the latest + uncommitted changes. Otherwise, anything successfully parsed by + :manpage:`git-rev-parse(1)` is a valid option. + + :param allow_unsafe_options: + Allow unsafe options in revision argument, like ``--output`` or ``--contents``. :return: list: [git.Commit, list: []] - A list of lists associating a Commit object with a list of lines that - changed within the given commit. The Commit objects will be given in order - of appearance. + A list of lists associating a :class:`~git.objects.commit.Commit` object + with a list of lines that changed within the given commit. The + :class:`~git.objects.commit.Commit` objects will be given in order of + appearance. """ if incremental: - return self.blame_incremental(rev, file, **kwargs) - rev_opts = rev_opts or [] - data: bytes = self.git.blame(rev, *rev_opts, "--", file, p=True, stdout_as_string=False, **kwargs) + return self.blame_incremental(rev, file, allow_unsafe_options=allow_unsafe_options, **kwargs) + rev_opts_list = list(rev_opts or []) + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([rev, rev_opts_list], kwargs), + unsafe_options=self.unsafe_git_blame_options, + clusterable_short_options="46bceflnpqstvw", + ) + data: bytes = self.git.blame(rev, *rev_opts_list, "--", file, p=True, stdout_as_string=False, **kwargs) commits: Dict[str, Commit] = {} blames: List[List[Commit | List[str | bytes] | None]] = [] @@ -1085,6 +1311,7 @@ class InfoTD(TypedDict, total=False): keepends = True for line_bytes in data.splitlines(keepends): + line_str = "" try: line_str = line_bytes.rstrip().decode(defenc) except UnicodeDecodeError: @@ -1092,10 +1319,11 @@ class InfoTD(TypedDict, total=False): parts = [] is_binary = True else: - # As we don't have an idea when the binary data ends, as it could contain multiple newlines - # in the process. So we rely on being able to decode to tell us what it is. - # This can absolutely fail even on text files, but even if it does, we should be fine treating it - # as binary instead. + # As we don't have an idea when the binary data ends, as it could + # contain multiple newlines in the process. So we rely on being able to + # decode to tell us what it is. This can absolutely fail even on text + # files, but even if it does, we should be fine treating it as binary + # instead. parts = self.re_whitespace.split(line_str, 1) firstpart = parts[0] is_binary = False @@ -1176,10 +1404,12 @@ class InfoTD(TypedDict, total=False): line = line_str else: line = line_bytes - # NOTE: We are actually parsing lines out of binary data, which can lead to the - # binary being split up along the newline separator. We will append this to the - # blame we are currently looking at, even though it should be concatenated with - # the last line we have seen. + # NOTE: We are actually parsing lines out of binary + # data, which can lead to the binary being split up + # along the newline separator. We will append this + # to the blame we are currently looking at, even + # though it should be concatenated with the last + # line we have seen. blames[-1][1].append(line) info = {"id": sha} @@ -1196,34 +1426,47 @@ def init( mkdir: bool = True, odbt: Type[GitCmdObjectDB] = GitCmdObjectDB, expand_vars: bool = True, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> "Repo": """Initialize a git repository at the given path if specified. :param path: - The full path to the repo (traditionally ends with /.git) or None in - which case the repository will be created in the current working directory + The full path to the repo (traditionally ends with ``/.git``). Or + ``None``, in which case the repository will be created in the current + working directory. :param mkdir: - If specified, will create the repository directory if it doesn't - already exist. Creates the directory with a mode=0755. + If specified, will create the repository directory if it doesn't already + exist. Creates the directory with a mode=0755. Only effective if a path is explicitly given. :param odbt: - Object DataBase type - a type which is constructed by providing - the directory containing the database objects, i.e. .git/objects. - It will be used to access all object data. + Object DataBase type - a type which is constructed by providing the + directory containing the database objects, i.e. ``.git/objects``. It will be + used to access all object data. :param expand_vars: - If specified, environment variables will not be escaped. This - can lead to information disclosure, allowing attackers to - access the contents of environment variables. + If specified, environment variables will not be escaped. This can lead to + information disclosure, allowing attackers to access the contents of + environment variables. + + :param allow_unsafe_options: + Allow unsafe options to be used, such as ``--template`` and + ``--separate-git-dir``. :param kwargs: - Keyword arguments serving as additional options to the git-init command. + Keyword arguments serving as additional options to the + :manpage:`git-init(1)` command. - :return: ``git.Repo`` (the newly created repo) + :return: + :class:`Repo` (the newly created repo) """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=cls.unsafe_git_init_options, + ) if path: path = expand_path(path, expand_vars) if mkdir and path and not osp.exists(path): @@ -1249,9 +1492,9 @@ def _clone( ) -> "Repo": odbt = kwargs.pop("odbt", odb_default_type) - # When pathlib.Path or other classbased path is passed - if not isinstance(path, str): - path = str(path) + # url may be a path and this has no effect if it is a string + url = os.fspath(url) + path = os.fspath(path) ## A bug win cygwin's Git, when `--bare` or `--separate-git-dir` # it prepends the cwd or(?) the `url` into the `path, so:: @@ -1267,17 +1510,21 @@ def _clone( if multi_options: multi = shlex.split(" ".join(multi_options)) + clone_url = Git.polish_url(url, expand_vars=False) if not allow_unsafe_protocols: - Git.check_unsafe_protocols(str(url)) + Git.check_unsafe_protocols(clone_url) if not allow_unsafe_options: - Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=cls.unsafe_git_clone_options) - if not allow_unsafe_options and multi_options: - Git.check_unsafe_options(options=multi_options, unsafe_options=cls.unsafe_git_clone_options) + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=cls.unsafe_git_clone_options, + ) + if not allow_unsafe_options and multi: + Git.check_unsafe_options(options=multi, unsafe_options=cls.unsafe_git_clone_options) proc = git.clone( multi, "--", - Git.polish_url(str(url)), + clone_url, clone_path, with_extended_output=True, as_process=True, @@ -1298,7 +1545,7 @@ def _clone( cmdline = getattr(proc, "args", "") cmdline = remove_password_if_present(cmdline) - log.debug("Cmd(%s)'s unused stdout: %s", cmdline, stdout) + _logger.debug("Cmd(%s)'s unused stdout: %s", cmdline, stdout) finalize_process(proc, stderr=stderr) # Our git command could have a different working dir than our actual @@ -1311,14 +1558,13 @@ def _clone( # Retain env values that were passed to _clone(). repo.git.update_environment(**git.environment()) - # Adjust remotes - there may be operating systems which use backslashes, - # These might be given as initial paths, but when handling the config file - # that contains the remote from which we were clones, git stops liking it - # as it will escape the backslashes. Hence we undo the escaping just to be - # sure. + # Adjust remotes - there may be operating systems which use backslashes, These + # might be given as initial paths, but when handling the config file that + # contains the remote from which we were clones, git stops liking it as it will + # escape the backslashes. Hence we undo the escaping just to be sure. if repo.remotes: with repo.remotes[0].config_writer as writer: - writer.set_value("url", Git.polish_url(repo.remotes[0].url)) + writer.set_value("url", Git.polish_url(repo.remotes[0].url, expand_vars=False)) # END handle remote repo return repo @@ -1333,28 +1579,47 @@ def clone( ) -> "Repo": """Create a clone from this repository. - :param path: The full path of the new repo (traditionally ends with - ``./.git``). - :param progress: See :meth:`git.remote.Remote.push`. - :param multi_options: A list of Clone options that can be provided multiple times. + :param path: + The full path of the new repo (traditionally ends with ``./.git``). + + :param progress: + See :meth:`Remote.push `. + + :param multi_options: + A list of :manpage:`git-clone(1)` options that can be provided multiple + times. + One option per list item which is passed exactly as specified to clone. - For example: ['--config core.filemode=false', '--config core.ignorecase', - '--recurse-submodule=repo1_path', '--recurse-submodule=repo2_path'] - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. + For example:: + + [ + "--config core.filemode=false", + "--config core.ignorecase", + "--recurse-submodule=repo1_path", + "--recurse-submodule=repo2_path", + ] + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. + :param kwargs: - * odbt = ObjectDatabase Type, allowing to determine the object database - implementation used by the returned Repo instance. - * All remaining keyword arguments are given to the git-clone command. + * ``odbt`` = ObjectDatabase Type, allowing to determine the object database + implementation used by the returned :class:`Repo` instance. + * All remaining keyword arguments are given to the :manpage:`git-clone(1)` + command. - :return: :class:`Repo` (the newly cloned repo) + :return: + :class:`Repo` (the newly cloned repo) """ return self._clone( self.git, self.common_dir, path, type(self.odb), - progress, + progress, # type: ignore[arg-type] multi_options, allow_unsafe_protocols=allow_unsafe_protocols, allow_unsafe_options=allow_unsafe_options, @@ -1375,29 +1640,37 @@ def clone_from( ) -> "Repo": """Create a clone from the given URL. - :param url: Valid git url, see http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS + :param url: + Valid git url, see: https://git-scm.com/docs/git-clone#URLS - :param to_path: Path to which the repository should be cloned to. + :param to_path: + Path to which the repository should be cloned to. - :param progress: See :meth:`git.remote.Remote.push`. + :param progress: + See :meth:`Remote.push `. - :param env: Optional dictionary containing the desired environment variables. + :param env: + Optional dictionary containing the desired environment variables. - Note: Provided variables will be used to update the execution - environment for `git`. If some variable is not specified in `env` - and is defined in `os.environ`, value from `os.environ` will be used. - If you want to unset some variable, consider providing empty string - as its value. + Note: Provided variables will be used to update the execution environment + for ``git``. If some variable is not specified in `env` and is defined in + :attr:`os.environ`, value from :attr:`os.environ` will be used. If you want + to unset some variable, consider providing empty string as its value. - :param multi_options: See :meth:`clone` method. + :param multi_options: + See the :meth:`clone` method. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. - :param kwargs: See the :meth:`clone` method. + :param kwargs: + See the :meth:`clone` method. - :return: :class:`Repo` instance pointing to the cloned directory. + :return: + :class:`Repo` instance pointing to the cloned directory. """ git = cls.GitCommandWrapperType(os.getcwd()) if env is not None: @@ -1407,7 +1680,7 @@ def clone_from( url, to_path, GitCmdObjectDB, - progress, + progress, # type: ignore[arg-type] multi_options, allow_unsafe_protocols=allow_unsafe_protocols, allow_unsafe_options=allow_unsafe_options, @@ -1419,31 +1692,54 @@ def archive( ostream: Union[TextIO, BinaryIO], treeish: Optional[str] = None, prefix: Optional[str] = None, + allow_unsafe_options: bool = False, + allow_unsafe_protocols: bool = False, **kwargs: Any, ) -> Repo: """Archive the tree at the given revision. - :param ostream: file compatible stream object to which the archive will be written as bytes. + :param ostream: + File-compatible stream object to which the archive will be written as bytes. + + :param treeish: + The treeish name/id, defaults to active branch. + + :param prefix: + The optional prefix to prepend to each filename in the archive. - :param treeish: is the treeish name/id, defaults to active branch. + :param kwargs: + Additional arguments passed to :manpage:`git-archive(1)`: - :param prefix: is the optional prefix to prepend to each filename in the archive. + * Use the ``format`` argument to define the kind of format. Use specialized + ostreams to write any format supported by Python. + * You may specify the special ``path`` keyword, which may either be a + repository-relative path to a directory or file to place into the archive, + or a list or tuple of multiple paths. - :param kwargs: Additional arguments passed to git-archive: + :param allow_unsafe_options: + Allow unsafe options, like ``--exec`` or ``--output``. - * Use the 'format' argument to define the kind of format. Use - specialized ostreams to write any format supported by python. - * You may specify the special **path** keyword, which may either be a repository-relative - path to a directory or file to place into the archive, or a list or tuple of multiple paths. + :param allow_unsafe_protocols: + Allow unsafe protocols to be used in ``remote``, like ``ext``. - :raise GitCommandError: If something went wrong. + :raise git.exc.GitCommandError: + If something went wrong. - :return: self + :return: + self """ if treeish is None: treeish = self.head.commit if prefix and "prefix" not in kwargs: kwargs["prefix"] = prefix + remote = kwargs.get("remote") + if not allow_unsafe_protocols and remote is not None: + Git.check_unsafe_protocols(str(remote)) + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_archive_options, + ) kwargs["output_stream"] = ostream path = kwargs.pop("path", []) path = cast(Union[PathLike, List[PathLike], Tuple[PathLike, ...]], path) @@ -1455,10 +1751,14 @@ def archive( def has_separate_working_tree(self) -> bool: """ - :return: True if our git_dir is not at the root of our working_tree_dir, but a .git file with a - platform agnositic symbolic link. Our git_dir will be wherever the .git file points to. + :return: + True if our :attr:`git_dir` is not at the root of our + :attr:`working_tree_dir`, but a ``.git`` file with a platform-agnostic + symbolic link. Our :attr:`git_dir` will be wherever the ``.git`` file points + to. - :note: bare repositories will always return False here + :note: + Bare repositories will always return ``False`` here. """ if self.bare: return False @@ -1475,12 +1775,14 @@ def __repr__(self) -> str: def currently_rebasing_on(self) -> Commit | None: """ - :return: The commit which is currently being replayed while rebasing. + :return: + The commit which is currently being replayed while rebasing. - None if we are not currently rebasing. + ``None`` if we are not currently rebasing. """ - if self.git_dir: - rebase_head_file = osp.join(self.git_dir, "REBASE_HEAD") + if not self.git_dir: + return None + rebase_head_file = osp.join(self.git_dir, "REBASE_HEAD") if not osp.isfile(rebase_head_file): return None with open(rebase_head_file, "rt") as f: diff --git a/git/repo/fun.py b/git/repo/fun.py index 63bcfdfc7..66e7eba69 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -5,49 +5,50 @@ from __future__ import annotations +__all__ = [ + "rev_parse", + "is_git_dir", + "touch", + "find_submodule_git_dir", + "name_to_object", + "short_to_long", + "deref_tag", + "to_commit", + "find_worktree_git_dir", +] + import os -import stat +import os.path as osp from pathlib import Path +import re +import stat from string import digits +from gitdb.exc import BadName, BadObject + +from git.cmd import Git from git.exc import WorkTreeRepositoryUnsupported from git.objects import Object +from git.objects.util import parse_date from git.refs import SymbolicReference -from git.util import hex_to_bin, bin_to_hex, cygpath -from gitdb.exc import ( - BadObject, - BadName, -) - -import os.path as osp -from git.cmd import Git +from git.util import cygpath, bin_to_hex, hex_to_bin # Typing ---------------------------------------------------------------------- -from typing import Union, Optional, cast, TYPE_CHECKING -from git.types import Commit_ish +from typing import Iterator, Optional, TYPE_CHECKING, Tuple, Union, cast, overload + +from git.types import AnyGitObject, Literal, PathLike if TYPE_CHECKING: - from git.types import PathLike - from .base import Repo from git.db import GitCmdObjectDB + from git.objects import Commit from git.refs.reference import Reference - from git.objects import Commit, TagObject, Blob, Tree + from git.refs.log import RefLog, RefLogEntry from git.refs.tag import Tag -# ---------------------------------------------------------------------------- + from .base import Repo -__all__ = ( - "rev_parse", - "is_git_dir", - "touch", - "find_submodule_git_dir", - "name_to_object", - "short_to_long", - "deref_tag", - "to_commit", - "find_worktree_git_dir", -) +# ---------------------------------------------------------------------------- def touch(filename: str) -> str: @@ -56,14 +57,14 @@ def touch(filename: str) -> str: return filename -def is_git_dir(d: "PathLike") -> bool: - """This is taken from the git setup.c:is_git_directory - function. +def is_git_dir(d: PathLike) -> bool: + """This is taken from the git setup.c:is_git_directory function. - @throws WorkTreeRepositoryUnsupported if it sees a worktree directory. It's quite hacky to do that here, - but at least clearly indicates that we don't support it. - There is the unlikely danger to throw if we see directories which just look like a worktree dir, - but are none.""" + :raise git.exc.WorkTreeRepositoryUnsupported: + If it sees a worktree directory. It's quite hacky to do that here, but at least + clearly indicates that we don't support it. There is the unlikely danger to + throw if we see directories which just look like a worktree dir, but are none. + """ if osp.isdir(d): if (osp.isdir(osp.join(d, "objects")) or "GIT_OBJECT_DIRECTORY" in os.environ) and osp.isdir( osp.join(d, "refs") @@ -79,7 +80,7 @@ def is_git_dir(d: "PathLike") -> bool: return False -def find_worktree_git_dir(dotgit: "PathLike") -> Optional[str]: +def find_worktree_git_dir(dotgit: PathLike) -> Optional[str]: """Search for a gitdir for this worktree.""" try: statbuf = os.stat(dotgit) @@ -98,7 +99,7 @@ def find_worktree_git_dir(dotgit: "PathLike") -> Optional[str]: return None -def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]: +def find_submodule_git_dir(d: PathLike) -> Optional[PathLike]: """Search for a submodule repo.""" if is_git_dir(d): return d @@ -107,15 +108,15 @@ def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]: with open(d) as fp: content = fp.read().rstrip() except IOError: - # it's probably not a file + # It's probably not a file. pass else: if content.startswith("gitdir: "): path = content[8:] if Git.is_cygwin(): - ## Cygwin creates submodules prefixed with `/cygdrive/...` suffixes. - # Cygwin git understands Cygwin paths much better than Windows ones + # Cygwin creates submodules prefixed with `/cygdrive/...`. + # Cygwin git understands Cygwin paths much better than Windows ones. # Also the Cygwin tests are assuming Cygwin paths. path = cygpath(path) if not osp.isabs(path): @@ -126,9 +127,14 @@ def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]: def short_to_long(odb: "GitCmdObjectDB", hexsha: str) -> Optional[bytes]: - """:return: long hexadecimal sha1 from the given less-than-40 byte hexsha - or None if no candidate could be found. - :param hexsha: hexsha with less than 40 byte""" + """ + :return: + Long hexadecimal sha1 from the given less than 40 byte hexsha, or ``None`` if no + candidate could be found. + + :param hexsha: + hexsha with less than 40 bytes. + """ try: return bin_to_hex(odb.partial_to_complete_sha_hex(hexsha)) except BadObject: @@ -136,29 +142,56 @@ def short_to_long(odb: "GitCmdObjectDB", hexsha: str) -> Optional[bytes]: # END exception handling -def name_to_object( - repo: "Repo", name: str, return_ref: bool = False -) -> Union[SymbolicReference, "Commit", "TagObject", "Blob", "Tree"]: +def _describe_to_long(repo: "Repo", name: str) -> Optional[bytes]: + """Resolve git-describe style names to the abbreviated object they contain.""" + match = re.match(r"^.+-\d+-g([0-9A-Fa-f]{4,40})(?:-dirty)?$", name) + if match is None: + match = re.match(r"^.+-g([0-9A-Fa-f]{4,40})(?:-dirty)?$", name) + if match is None: + match = re.match(r"^([0-9A-Fa-f]{4,40})-dirty$", name) + if match is None: + return None + # END handle match + + hexsha = match.group(1) + if len(hexsha) == 40: + return hexsha.encode("ascii") + return short_to_long(repo.odb, hexsha) + + +@overload +def name_to_object(repo: "Repo", name: str, return_ref: Literal[False] = ...) -> AnyGitObject: ... + + +@overload +def name_to_object(repo: "Repo", name: str, return_ref: Literal[True]) -> Union[AnyGitObject, SymbolicReference]: ... + + +def name_to_object(repo: "Repo", name: str, return_ref: bool = False) -> Union[AnyGitObject, SymbolicReference]: """ - :return: object specified by the given name, hexshas ( short and long ) - as well as references are supported - :param return_ref: if name specifies a reference, we will return the reference - instead of the object. Otherwise it will raise BadObject or BadName + :return: + Object specified by the given name - hexshas (short and long) as well as + references are supported. + + :param return_ref: + If ``True``, and name specifies a reference, we will return the reference + instead of the object. Otherwise it will raise :exc:`~gitdb.exc.BadObject` or + :exc:`~gitdb.exc.BadName`. """ hexsha: Union[None, str, bytes] = None - # is it a hexsha ? Try the most common ones, which is 7 to 40 + # Is it a hexsha? Try the most common ones, which is 7 to 40. if repo.re_hexsha_shortened.match(name): if len(name) != 40: - # find long sha for short sha + # Find long sha for short sha. hexsha = short_to_long(repo.odb, name) else: hexsha = name # END handle short shas # END find sha if it matches - # if we couldn't find an object for what seemed to be a short hexsha - # try to find it as reference anyway, it could be named 'aaa' for instance + # If we couldn't find an object for what seemed to be a short hexsha, try to find it + # as reference anyway, it could be named 'aaa' for instance. if hexsha is None: for base in ( "%s", @@ -179,12 +212,16 @@ def name_to_object( # END for each base # END handle hexsha - # didn't find any ref, this is an error + if hexsha is None: + hexsha = _describe_to_long(repo, name) + # END handle describe output + + # Didn't find any ref, this is an error. if return_ref: raise BadObject("Couldn't find reference named %r" % name) # END handle return ref - # tried everything ? fail + # Tried everything ? fail. if hexsha is None: raise BadName(name) # END assert hexsha was found @@ -192,8 +229,8 @@ def name_to_object( return Object.new_from_sha(repo, hex_to_bin(hexsha)) -def deref_tag(tag: "Tag") -> "TagObject": - """Recursively dereference a tag and return the resulting object""" +def deref_tag(tag: "Tag") -> AnyGitObject: + """Recursively dereference a tag and return the resulting object.""" while True: try: tag = tag.object @@ -203,8 +240,8 @@ def deref_tag(tag: "Tag") -> "TagObject": return tag -def to_commit(obj: Object) -> Union["Commit", "TagObject"]: - """Convert the given object to a commit if possible and return it""" +def to_commit(obj: Object) -> "Commit": + """Convert the given object to a commit if possible and return it.""" if obj.type == "tag": obj = deref_tag(obj) @@ -214,144 +251,463 @@ def to_commit(obj: Object) -> Union["Commit", "TagObject"]: return obj -def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: +def _object_from_hexsha(repo: "Repo", hexsha: str) -> AnyGitObject: + return Object.new_from_sha(repo, hex_to_bin(hexsha)) + + +def _current_reflog_ref(repo: "Repo") -> SymbolicReference: + try: + return repo.head.ref + except TypeError: + return repo.head + # END handle detached head + + +def _common_reflog_path(repo: "Repo", ref: SymbolicReference) -> Optional[str]: + if repo.common_dir == repo.git_dir: + return None + # END handle normal repository + return SymbolicReference._get_validated_path(osp.join(repo.common_dir, "logs"), ref.path) + + +def _ref_log(repo: "Repo", ref: SymbolicReference) -> "RefLog": + try: + return ref.log() + except FileNotFoundError: + common_path = _common_reflog_path(repo, ref) + if common_path and osp.isfile(common_path): + from git.refs.log import RefLog + + return RefLog.from_file(common_path) + # END handle linked-worktree branch logs + try: + if ref.path == repo.head.ref.path: + return repo.head.log() + # END handle linked-worktree current branch logs + except TypeError: + pass + # END handle detached head + raise + # END handle missing branch log + + +def _ref_log_entry(repo: "Repo", ref: SymbolicReference, index: int) -> "RefLogEntry": + try: + return ref.log_entry(index) + except FileNotFoundError: + common_path = _common_reflog_path(repo, ref) + if common_path and osp.isfile(common_path): + from git.refs.log import RefLog + + return RefLog.entry_at(common_path, index) + # END handle linked-worktree branch logs + try: + if ref.path == repo.head.ref.path: + return repo.head.log_entry(index) + # END handle linked-worktree current branch logs + except TypeError: + pass + # END handle detached head + raise + # END handle missing branch log + + +def _find_reflog_entry_by_date(repo: "Repo", ref: SymbolicReference, spec: str) -> str: + try: + timestamp, _offset = parse_date(spec) + except ValueError as e: + raise NotImplementedError("Support for additional @{...} modes not implemented") from e + # END handle unsupported dates + log = _ref_log(repo, ref) + if not log: + raise IndexError("Invalid revlog date: %s" % spec) + # END handle empty log + + for entry in reversed(log): + if entry.time[0] <= timestamp: + return entry.newhexsha + # END found candidate + # END for each entry + return log[0].newhexsha + + +def _previous_checked_out_branch(repo: "Repo", nth: int) -> AnyGitObject: + if nth <= 0: + raise ValueError("Invalid previous checkout selector: -%i" % nth) + # END handle invalid input + + seen = 0 + for entry in reversed(_ref_log(repo, repo.head)): + message = entry.message or "" + prefix = "checkout: moving from " + if not message.startswith(prefix): + continue + # END skip non-checkouts + + previous_branch = message[len(prefix) :].split(" to ", 1)[0] + seen += 1 + if seen == nth: + return name_to_object(repo, previous_branch) + # END found selector + # END for each entry + raise IndexError("Invalid previous checkout selector: -%i" % nth) + + +def _tracking_branch_object(repo: "Repo", ref: Optional[SymbolicReference]) -> AnyGitObject: + from git.refs.head import Head + + if ref is None: + try: + head = repo.active_branch + except TypeError as e: + raise BadName("@{upstream}") from e + elif isinstance(ref, Head): + head = ref + elif os.fspath(ref.path).startswith("refs/heads/"): + head = Head(repo, ref.path) + else: + raise BadName("%s@{upstream}" % ref.name) + # END handle head + + tracking_branch = head.tracking_branch() + if tracking_branch is None: + raise BadName("%s@{upstream}" % head.name) + # END handle missing upstream + return tracking_branch.commit + + +def _apply_reflog(repo: "Repo", ref: Optional[SymbolicReference], content: str) -> AnyGitObject: + if content.startswith("+"): + content = content[1:] + # END handle explicit positive sign + + if content.startswith("-"): + if ref is not None: + raise ValueError("Previous checkout selectors do not take an explicit ref") + if content == "-0": + raise ValueError("Negative zero is invalid in reflog selector") + # END handle invalid negative zero + try: + return _previous_checked_out_branch(repo, int(content[1:])) + except ValueError as e: + raise ValueError("Invalid previous checkout selector: %s" % content) from e + # END handle previous checkout branch + + content_lower = content.lower() + if content_lower in ("u", "upstream", "push"): + return _tracking_branch_object(repo, ref) + # END handle sibling branches + + ref = ref or _current_reflog_ref(repo) + try: + entry_no = int(content) + except ValueError: + hexsha = _find_reflog_entry_by_date(repo, ref, content) + else: + if entry_no >= 100000000: + hexsha = _find_reflog_entry_by_date(repo, ref, "%s +0000" % entry_no) + elif entry_no == 0: + return ref.commit + else: + try: + entry = _ref_log_entry(repo, ref, -(entry_no + 1)) + except IndexError as e: + raise IndexError("Invalid revlog index: %i" % entry_no) from e + # END handle index out of bound + hexsha = entry.newhexsha + # END handle offset or date-like timestamp + # END handle content + return _object_from_hexsha(repo, hexsha) + + +def _find_closing_brace(rev: str, start: int) -> int: + depth = 1 + escaped = False + for idx in range(start + 1, len(rev)): + char = rev[idx] + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return idx + # END found end + # END handle char + # END for each char + raise ValueError("Missing closing brace to define type in %s" % rev) + + +def _parse_search(pattern: str) -> Tuple[str, bool]: + if not pattern: + raise ValueError("Revision search requires a pattern") + # END handle empty pattern + + if pattern.startswith("!-"): + return pattern[2:], True + if pattern.startswith("!!"): + return pattern[1:], False + if pattern.startswith("!"): + raise ValueError("Need one character after /!, typically -") + return pattern, False + + +def _unescape_braced_regex(pattern: str) -> str: + out = [] + idx = 0 + while idx < len(pattern): + char = pattern[idx] + if char == "\\" and idx + 1 < len(pattern): + next_char = pattern[idx + 1] + if next_char in "{}\\": + out.append(next_char) + else: + out.append(char) + out.append(next_char) + # END handle escaped char + idx += 2 + continue + # END handle backslash + out.append(char) + idx += 1 + # END for each char + return "".join(out) + + +def _find_commit_by_message( + repo: "Repo", rev: Optional[AnyGitObject], pattern: str, braced: bool = False +) -> AnyGitObject: + pattern, negated = _parse_search(_unescape_braced_regex(pattern) if braced else pattern) + try: + regex = re.compile(pattern) + except re.error as e: + raise ValueError("Invalid commit message regex %r" % pattern) from e + # END handle invalid regex + if rev is None: + commits = _all_ref_commits(repo) + else: + commits = _reachable_commits([to_commit(cast(Object, rev))]) + # END handle starting point + + for commit in commits: + message = commit.message + if isinstance(message, bytes): + message = message.decode(commit.encoding, "replace") + # END handle bytes message + matches = regex.search(message or "") is not None + if matches != negated: + return commit + # END found commit + # END for each commit + raise BadName("No commit found matching message pattern %r" % pattern) + + +def _all_ref_commits(repo: "Repo") -> Iterator["Commit"]: + starts = [] + for ref in repo.references: + try: + starts.append(to_commit(cast(Object, ref.object))) + except (BadName, ValueError): + pass + # END skip refs that do not point to commits + # END for each ref + try: + starts.append(repo.head.commit) + except ValueError: + pass + # END handle unborn head + return _reachable_commits(starts) + + +def _reachable_commits(starts: list["Commit"]) -> Iterator["Commit"]: + seen = set() + pending = starts[:] + while pending: + pending.sort(key=lambda commit: commit.committed_date, reverse=True) + commit = pending.pop(0) + if commit.binsha in seen: + continue + # END skip seen commit + seen.add(commit.binsha) + yield commit + pending.extend(commit.parents) + # END while commits remain + + +def _index_lookup(repo: "Repo", spec: str) -> AnyGitObject: + if not spec: + raise ValueError("':' must be followed by a path") + # END handle empty lookup + + stage = 0 + path = spec + if len(spec) >= 2 and spec[1] == ":" and spec[0] in "0123": + stage = int(spec[0]) + path = spec[2:] + # END handle stage + + try: + return repo.index.entries[(path, stage)].to_blob(repo) + except KeyError as e: + raise BadName("Path %r did not exist in the index at stage %i" % (path, stage)) from e + + +def _tree_lookup(obj: AnyGitObject, path: str) -> AnyGitObject: + if obj.type != "tree": + obj = to_commit(cast(Object, obj)).tree + # END get tree + if not path: + return obj + return obj[path] + + +def _peel(obj: AnyGitObject, output_type: str, repo: "Repo", rev: str) -> AnyGitObject: + if output_type.startswith("/"): + return _find_commit_by_message(repo, obj, output_type[1:], braced=True) + if output_type == "": + return deref_tag(obj) if obj.type == "tag" else obj + if output_type == "object": + return obj + if output_type == "commit": + return to_commit(cast(Object, obj)) + if output_type == "tree": + return to_commit(cast(Object, obj)).tree if obj.type != "tree" else obj + if output_type == "blob": + obj = deref_tag(obj) if obj.type == "tag" else obj + if obj.type == output_type: + return obj + # END handle matching type + raise ValueError("Could not accommodate requested object type %r, got %s" % (output_type, obj.type)) + if output_type == "tag": + if obj.type == output_type: + return obj + # END handle matching type + raise ValueError("Could not accommodate requested object type %r, got %s" % (output_type, obj.type)) + # END handle known types + raise ValueError("Invalid output type: %s ( in %s )" % (output_type, rev)) + + +def _first_rev_token(rev: str) -> Optional[int]: + for idx, char in enumerate(rev): + if char in "^~:": + return idx + if char == "@": + next_char = rev[idx + 1] if idx + 1 < len(rev) else None + if idx == 0 and next_char in (None, "^", "~", ":", "{"): + return idx + if next_char == "{": + return idx + # END handle reflog selector + # END handle at symbol + # END for each char + return None + + +def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: + """Parse a revision string. Like :manpage:`git-rev-parse(1)`. + + :return: + `~git.objects.base.Object` at the given revision. + + This may be any type of git object: + + * :class:`Commit ` + * :class:`TagObject ` + * :class:`Tree ` + * :class:`Blob ` + + :param rev: + :manpage:`git-rev-parse(1)`-compatible revision specification as string. + Please see :manpage:`git-rev-parse(1)` for details. + + :raise gitdb.exc.BadObject: + If the given revision could not be found. + + :raise ValueError: + If `rev` couldn't be parsed. + + :raise IndexError: + If an invalid reflog index is specified. """ - :return: Object at the given revision, either Commit, Tag, Tree or Blob - :param rev: git-rev-parse compatible revision specification as string, please see - http://www.kernel.org/pub/software/scm/git/docs/git-rev-parse.html - for details - :raise BadObject: if the given revision could not be found - :raise ValueError: If rev couldn't be parsed - :raise IndexError: If invalid reflog index is specified""" - - # colon search mode ? if rev.startswith(":/"): - # colon search mode - raise NotImplementedError("commit by message search ( regex )") - # END handle search + return _find_commit_by_message(repo, None, rev[2:]) + if rev.startswith(":"): + return _index_lookup(repo, rev[1:]) + # END handle top-level colon modes - obj: Union[Commit_ish, "Reference", None] = None + obj: Optional[AnyGitObject] = None ref = None - output_type = "commit" - start = 0 - parsed_to = 0 lr = len(rev) - while start < lr: - if rev[start] not in "^~:@": - start += 1 - continue - # END handle start + first_token = _first_rev_token(rev) + if first_token is None: + return name_to_object(repo, rev) + # END handle plain name + + if first_token == 0: + if rev[0] != "@": + raise ValueError("Revision specifier must start with an object name: %s" % rev) + # END handle invalid leading token + ref = _current_reflog_ref(repo) + obj = ref.commit + start = 0 if rev.startswith("@{") else 1 + else: + if rev[first_token] == "@": + ref = cast("Reference", name_to_object(repo, rev[:first_token], return_ref=True)) + obj = ref.commit + else: + obj = name_to_object(repo, rev[:first_token]) + # END handle anchor + start = first_token + # END initialize anchor + while start < lr: token = rev[start] - if obj is None: - # token is a rev name - if start == 0: - ref = repo.head.ref - else: - if token == "@": - ref = cast("Reference", name_to_object(repo, rev[:start], return_ref=True)) - else: - obj = cast(Commit_ish, name_to_object(repo, rev[:start])) - # END handle token - # END handle refname - else: - assert obj is not None + if token == "@": + if start + 1 >= lr or rev[start + 1] != "{": + raise ValueError("Invalid @ token in revision specifier: %s" % rev) + # END handle invalid @ + end = _find_closing_brace(rev, start + 1) + obj = _apply_reflog(repo, ref if first_token != 0 and start == first_token else None, rev[start + 2 : end]) + ref = None + start = end + 1 + continue + # END handle reflog - if ref is not None: - obj = cast("Commit", ref.commit) - # END handle ref - # END initialize obj on first token + if token == ":": + return _tree_lookup(obj, rev[start + 1 :]) + # END handle path start += 1 - # try to parse {type} - if start < lr and rev[start] == "{": - end = rev.find("}", start) - if end == -1: - raise ValueError("Missing closing brace to define type in %s" % rev) - output_type = rev[start + 1 : end] # exclude brace - - # handle type - if output_type == "commit": - pass # default - elif output_type == "tree": - try: - obj = cast(Commit_ish, obj) - obj = to_commit(obj).tree - except (AttributeError, ValueError): - pass # error raised later - # END exception handling - elif output_type in ("", "blob"): - obj = cast("TagObject", obj) - if obj and obj.type == "tag": - obj = deref_tag(obj) - else: - # cannot do anything for non-tags - pass - # END handle tag - elif token == "@": - # try single int - assert ref is not None, "Require Reference to access reflog" - revlog_index = None - try: - # transform reversed index into the format of our revlog - revlog_index = -(int(output_type) + 1) - except ValueError as e: - # TODO: Try to parse the other date options, using parse_date - # maybe - raise NotImplementedError("Support for additional @{...} modes not implemented") from e - # END handle revlog index - - try: - entry = ref.log_entry(revlog_index) - except IndexError as e: - raise IndexError("Invalid revlog index: %i" % revlog_index) from e - # END handle index out of bound - - obj = Object.new_from_sha(repo, hex_to_bin(entry.newhexsha)) - - # make it pass the following checks - output_type = "" - else: - raise ValueError("Invalid output type: %s ( in %s )" % (output_type, rev)) - # END handle output type - - # empty output types don't require any specific type, its just about dereferencing tags - if output_type and obj and obj.type != output_type: - raise ValueError("Could not accommodate requested object type %r, got %s" % (output_type, obj.type)) - # END verify output type - - start = end + 1 # skip brace - parsed_to = start + if token == "^" and start < lr and rev[start] == "{": + end = _find_closing_brace(rev, start) + obj = _peel(obj, rev[start + 1 : end], repo, rev) + ref = None + start = end + 1 continue # END parse type - # try to parse a number num = 0 - if token != ":": - found_digit = False - while start < lr: - if rev[start] in digits: - num = num * 10 + int(rev[start]) - start += 1 - found_digit = True - else: - break - # END handle number - # END number parse loop - - # no explicit number given, 1 is the default - # It could be 0 though - if not found_digit: - num = 1 - # END set default num - # END number parsing only if non-blob mode - - parsed_to = start - # handle hierarchy walk + found_digit = False + while start < lr: + if rev[start] in digits: + num = num * 10 + int(rev[start]) + start += 1 + found_digit = True + else: + break + # END handle number + # END number parse loop + + if not found_digit: + num = 1 + # END set default num + try: - obj = cast(Commit_ish, obj) if token == "~": obj = to_commit(obj) for _ in range(num): @@ -359,35 +715,22 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: # END for each history item to walk elif token == "^": obj = to_commit(obj) - # must be n'th parent - if num: + if num == 0: + pass + else: obj = obj.parents[num - 1] - elif token == ":": - if obj.type != "tree": - obj = obj.tree - # END get tree type - obj = obj[rev[start:]] - parsed_to = lr + # END handle parent else: raise ValueError("Invalid token: %r" % token) # END end handle tag except (IndexError, AttributeError) as e: raise BadName( - f"Invalid revision spec '{rev}' - not enough " f"parent commits to reach '{token}{int(num)}'" + f"Invalid revision spec '{rev}' - not enough parent commits to reach '{token}{int(num)}'" ) from e # END exception handling # END parse loop - # still no obj ? Its probably a simple name - if obj is None: - obj = cast(Commit_ish, name_to_object(repo, rev)) - parsed_to = lr - # END handle simple name - if obj is None: raise ValueError("Revision specifier could not be parsed: %s" % rev) - if parsed_to != lr: - raise ValueError("Didn't consume complete rev spec %s, consumed part: %s" % (rev, rev[:parsed_to])) - return obj diff --git a/git/types.py b/git/types.py index efb393471..100fff43f 100644 --- a/git/types.py +++ b/git/types.py @@ -3,99 +3,261 @@ import os import sys -from typing import ( # noqa: F401 +from typing import ( + Any, + Callable, Dict, + List, NoReturn, + Optional, Sequence as Sequence, Tuple, - Union, - Any, - Optional, - Callable, TYPE_CHECKING, TypeVar, + Union, ) +import warnings if sys.version_info >= (3, 8): - from typing import ( # noqa: F401 + from typing import ( Literal, - TypedDict, Protocol, SupportsIndex as SupportsIndex, + TypedDict, runtime_checkable, ) else: - from typing_extensions import ( # noqa: F401 + from typing_extensions import ( Literal, + Protocol, SupportsIndex as SupportsIndex, TypedDict, - Protocol, runtime_checkable, ) -# if sys.version_info >= (3, 10): -# from typing import TypeGuard # noqa: F401 -# else: -# from typing_extensions import TypeGuard # noqa: F401 - -PathLike = Union[str, "os.PathLike[str]"] - if TYPE_CHECKING: - from git.repo import Repo from git.objects import Commit, Tree, TagObject, Blob + from git.repo import Repo - # from git.refs import SymbolicReference +PathLike = Union[str, "os.PathLike[str]"] +"""A :class:`str` (Unicode) based file or directory path.""" TBD = Any +"""Alias of :class:`~typing.Any`, when a type hint is meant to become more specific.""" + _T = TypeVar("_T") +"""Type variable used internally in GitPython.""" + +AnyGitObject = Union["Commit", "Tree", "TagObject", "Blob"] +"""Union of the :class:`~git.objects.base.Object`-based types that represent actual git +object types. + +As noted in :class:`~git.objects.base.Object`, which has further details, these are: + +* :class:`Blob ` +* :class:`Tree ` +* :class:`Commit ` +* :class:`TagObject ` + +Those GitPython classes represent the four git object types, per +:manpage:`gitglossary(7)`: + +* "blob": https://git-scm.com/docs/gitglossary#def_blob_object +* "tree object": https://git-scm.com/docs/gitglossary#def_tree_object +* "commit object": https://git-scm.com/docs/gitglossary#def_commit_object +* "tag object": https://git-scm.com/docs/gitglossary#def_tag_object + +For more general information on git objects and their types as git understands them: + +* "object": https://git-scm.com/docs/gitglossary#def_object +* "object type": https://git-scm.com/docs/gitglossary#def_object_type + +:note: + See also the :class:`Tree_ish` and :class:`Commit_ish` unions. +""" + +Tree_ish = Union["Commit", "Tree", "TagObject"] +"""Union of :class:`~git.objects.base.Object`-based types that are typically tree-ish. + +See :manpage:`gitglossary(7)` on "tree-ish": +https://git-scm.com/docs/gitglossary#def_tree-ish + +:note: + :class:`~git.objects.tree.Tree` and :class:`~git.objects.commit.Commit` are the + classes whose instances are all tree-ish. This union includes them, but also + :class:`~git.objects.tag.TagObject`, only **most** of whose instances are tree-ish. + Whether a particular :class:`~git.objects.tag.TagObject` peels (recursively + dereferences) to a tree or commit, rather than a blob, can in general only be known + at runtime. In practice, git tag objects are nearly always used for tagging commits, + and such tags are tree-ish because commits are tree-ish. + +:note: + See also the :class:`AnyGitObject` union of all four classes corresponding to git + object types. +""" + +Commit_ish = Union["Commit", "TagObject"] +"""Union of :class:`~git.objects.base.Object`-based types that are typically commit-ish. + +See :manpage:`gitglossary(7)` on "commit-ish": +https://git-scm.com/docs/gitglossary#def_commit-ish + +:note: + :class:`~git.objects.commit.Commit` is the only class whose instances are all + commit-ish. This union type includes :class:`~git.objects.commit.Commit`, but also + :class:`~git.objects.tag.TagObject`, only **most** of whose instances are + commit-ish. Whether a particular :class:`~git.objects.tag.TagObject` peels + (recursively dereferences) to a commit, rather than a tree or blob, can in general + only be known at runtime. In practice, git tag objects are nearly always used for + tagging commits, and such tags are of course commit-ish. + +:note: + See also the :class:`AnyGitObject` union of all four classes corresponding to git + object types. +""" + +GitObjectTypeString = Literal["commit", "tag", "blob", "tree"] +"""Literal strings identifying git object types and the +:class:`~git.objects.base.Object`-based types that represent them. + +See the :attr:`Object.type ` attribute. These are its +values in :class:`~git.objects.base.Object` subclasses that represent git objects. These +literals therefore correspond to the types in the :class:`AnyGitObject` union. + +These are the same strings git itself uses to identify its four object types. +See :manpage:`gitglossary(7)` on "object type": +https://git-scm.com/docs/gitglossary#def_object_type +""" + +if TYPE_CHECKING: + Lit_commit_ish = Literal["commit", "tag"] +"""Deprecated. Type of literal strings identifying typically-commitish git object types. + +Prior to a bugfix, this type had been defined more broadly. Any usage is in practice +ambiguous and likely to be incorrect. This type has therefore been made a static type +error to appear in annotations. It is preserved, with a deprecated status, to avoid +introducing runtime errors in code that refers to it, but it should not be used. + +Instead of this type: + +* For the type of the string literals associated with :class:`Commit_ish`, use + ``Literal["commit", "tag"]`` or create a new type alias for it. That is equivalent to + this type as currently defined (but usable in statically checked type annotations). + +* For the type of all four string literals associated with :class:`AnyGitObject`, use + :class:`GitObjectTypeString`. That is equivalent to the old definition of this type + prior to the bugfix (and is also usable in statically checked type annotations). +""" + + +def _getattr(name: str) -> Any: + if name != "Lit_commit_ish": + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + warnings.warn( + "Lit_commit_ish is deprecated. It is currently defined as " + '`Literal["commit", "tag"]`, which should be used in its place if desired. It ' + 'had previously been defined as `Literal["commit", "tag", "blob", "tree"]`, ' + "covering all four git object type strings including those that are never " + "commit-ish. For that, use the GitObjectTypeString type instead.", + DeprecationWarning, + stacklevel=2, + ) + return Literal["commit", "tag"] + + +if not TYPE_CHECKING: # Preserve static checking for undefined/misspelled attributes. + __getattr__ = _getattr + + +def __dir__() -> List[str]: + return [*globals(), "Lit_commit_ish"] -Tree_ish = Union["Commit", "Tree"] -Commit_ish = Union["Commit", "TagObject", "Blob", "Tree"] -Lit_commit_ish = Literal["commit", "tag", "blob", "tree"] # Config_levels --------------------------------------------------------- Lit_config_levels = Literal["system", "global", "user", "repository"] +"""Type of literal strings naming git configuration levels. + +These strings relate to which file a git configuration variable is in. +""" + +ConfigLevels_Tup = Tuple[Literal["system"], Literal["user"], Literal["global"], Literal["repository"]] +"""Static type of a tuple of the four strings representing configuration levels.""" # Progress parameter type alias ----------------------------------------- CallableProgress = Optional[Callable[[int, Union[str, float], Union[str, float, None], str], None]] +"""General type of a function or other callable used as a progress reporter for cloning. -# def is_config_level(inp: str) -> TypeGuard[Lit_config_levels]: -# # return inp in get_args(Lit_config_level) # only py >= 3.8 -# return inp in ("system", "user", "global", "repository") +This is the type of a function or other callable that reports the progress of a clone, +when passed as a ``progress`` argument to :meth:`Repo.clone ` +or :meth:`Repo.clone_from `. +:note: + Those :meth:`~git.repo.base.Repo.clone` and :meth:`~git.repo.base.Repo.clone_from` + methods also accept :meth:`~git.util.RemoteProgress` instances, including instances + of its :meth:`~git.util.CallableRemoteProgress` subclass. -ConfigLevels_Tup = Tuple[Literal["system"], Literal["user"], Literal["global"], Literal["repository"]] +:note: + Unlike objects that match this type, :meth:`~git.util.RemoteProgress` instances are + not directly callable, not even when they are instances of + :meth:`~git.util.CallableRemoteProgress`, which wraps a callable and forwards + information to it but is not itself callable. + +:note: + This type also allows ``None``, for cloning without reporting progress. +""" # ----------------------------------------------------------------------------------- def assert_never(inp: NoReturn, raise_error: bool = True, exc: Union[Exception, None] = None) -> None: - """For use in exhaustive checking of literal or Enum in if/else chain. + """For use in exhaustive checking of a literal or enum in if/else chains. - Should only be reached if all members not handled OR attempt to pass non-members through chain. + A call to this function should only be reached if not all members are handled, or if + an attempt is made to pass non-members through the chain. - If all members handled, type is Empty. Otherwise, will cause mypy error. + :param inp: + If all members are handled, the argument for `inp` will have the + :class:`~typing.Never`/:class:`~typing.NoReturn` type. + Otherwise, the type will mismatch and cause a mypy error. - If non-members given, should cause mypy error at variable creation. + :param raise_error: + If ``True``, will also raise :exc:`ValueError` with a general + "unhandled literal" message, or the exception object passed as `exc`. - If raise_error is True, will also raise AssertionError or the Exception passed to exc. + :param exc: + It not ``None``, this should be an already-constructed exception object, to be + raised if `raise_error` is ``True``. """ if raise_error: if exc is None: - raise ValueError(f"An unhandled Literal ({inp}) in an if/else chain was found") + raise ValueError(f"An unhandled literal ({inp!r}) in an if/else chain was found") else: raise exc class Files_TD(TypedDict): + """Dictionary with stat counts for the diff of a particular file. + + For the :class:`~git.util.Stats.files` attribute of :class:`~git.util.Stats` + objects. + """ + insertions: int deletions: int lines: int + change_type: str class Total_TD(TypedDict): + """Dictionary with total stats from any number of files. + + For the :class:`~git.util.Stats.total` attribute of :class:`~git.util.Stats` + objects. + """ + insertions: int deletions: int lines: int @@ -103,15 +265,21 @@ class Total_TD(TypedDict): class HSH_TD(TypedDict): + """Dictionary carrying the same information as a :class:`~git.util.Stats` object.""" + total: Total_TD files: Dict[PathLike, Files_TD] @runtime_checkable class Has_Repo(Protocol): + """Protocol for having a :attr:`repo` attribute, the repository to operate on.""" + repo: "Repo" @runtime_checkable class Has_id_attribute(Protocol): + """Protocol for having :attr:`_id_attribute_` used in iteration and traversal.""" + _id_attribute_: str diff --git a/git/util.py b/git/util.py index 5ccd2b04c..b0593feea 100644 --- a/git/util.py +++ b/git/util.py @@ -3,6 +3,32 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import sys + +__all__ = [ + "stream_copy", + "join_path", + "to_native_path_linux", + "join_path_native", + "Stats", + "IndexFileSHA1Writer", + "IterableObj", + "IterableList", + "BlockingLockFile", + "LockFile", + "Actor", + "get_user_id", + "assure_directory_exists", + "RemoteProgress", + "CallableRemoteProgress", + "rmtree", + "unbare_repo", + "HIDE_WINDOWS_KNOWN_ERRORS", +] + +if sys.platform == "win32": + __all__.append("to_native_path_windows") + from abc import abstractmethod import contextlib from functools import wraps @@ -10,17 +36,33 @@ import logging import os import os.path as osp -import pathlib +from pathlib import Path import platform import re import shutil import stat import subprocess -import sys import time from urllib.parse import urlsplit, urlunsplit import warnings +# NOTE: Unused imports can be improved now that CI testing has fully resumed. Some of +# these be used indirectly through other GitPython modules, which avoids having to write +# gitdb all the time in their imports. They are not in __all__, at least currently, +# because they could be removed or changed at any time, and so should not be considered +# conceptually public to code outside GitPython. Linters of course do not like it. +from gitdb.util import ( + LazyMixin, # noqa: F401 + LockedFD, # noqa: F401 + bin_to_hex, # noqa: F401 + file_contents_ro, # noqa: F401 + file_contents_ro_filepath, # noqa: F401 + hex_to_bin, # noqa: F401 + make_sha, + to_bin_sha, # noqa: F401 + to_hex_sha, # noqa: F401 +) + # typing --------------------------------------------------------- from typing import ( @@ -37,94 +79,51 @@ Pattern, Sequence, Tuple, + TYPE_CHECKING, TypeVar, Union, - TYPE_CHECKING, cast, overload, ) if TYPE_CHECKING: + from git.cmd import Git + from git.config import GitConfigParser, SectionConstraint from git.remote import Remote from git.repo.base import Repo - from git.config import GitConfigParser, SectionConstraint - from git import Git -from .types import ( +from git.types import ( + Files_TD, + Has_id_attribute, + HSH_TD, Literal, - SupportsIndex, - Protocol, - runtime_checkable, # because behind py version guards PathLike, - HSH_TD, + Protocol, + SupportsIndex, Total_TD, - Files_TD, # aliases - Has_id_attribute, + runtime_checkable, ) # --------------------------------------------------------------------- -from gitdb.util import ( # noqa: F401 # @IgnorePep8 - make_sha, - LockedFD, # @UnusedImport - file_contents_ro, # @UnusedImport - file_contents_ro_filepath, # @UnusedImport - LazyMixin, # @UnusedImport - to_hex_sha, # @UnusedImport - to_bin_sha, # @UnusedImport - bin_to_hex, # @UnusedImport - hex_to_bin, # @UnusedImport -) - T_IterableObj = TypeVar("T_IterableObj", bound=Union["IterableObj", "Has_id_attribute"], covariant=True) # So IterableList[Head] is subtype of IterableList[IterableObj]. -# NOTE: Some of the unused imports might be used/imported by others. -# Handle once test-cases are back up and running. -# Most of these are unused here, but are for use by git-python modules so these -# don't see gitdb all the time. Flake of course doesn't like it. -__all__ = [ - "stream_copy", - "join_path", - "to_native_path_linux", - "join_path_native", - "Stats", - "IndexFileSHA1Writer", - "IterableObj", - "IterableList", - "BlockingLockFile", - "LockFile", - "Actor", - "get_user_id", - "assure_directory_exists", - "RemoteProgress", - "CallableRemoteProgress", - "rmtree", - "unbare_repo", - "HIDE_WINDOWS_KNOWN_ERRORS", -] - -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) -def _read_win_env_flag(name: str, default: bool) -> bool: - """Read a boolean flag from an environment variable on Windows. +def _read_env_flag(name: str, default: bool) -> bool: + """Read a boolean flag from an environment variable. :return: - On Windows, the flag, or the ``default`` value if absent or ambiguous. - On all other operating systems, ``False``. - - :note: This only accesses the environment on Windows. + The flag, or the `default` value if absent or ambiguous. """ - if os.name != "nt": - return False - try: value = os.environ[name] except KeyError: return default - log.warning( + _logger.warning( "The %s environment variable is deprecated. Its effect has never been documented and changes without warning.", name, ) @@ -135,10 +134,23 @@ def _read_win_env_flag(name: str, default: bool) -> bool: return False if adjusted_value in {"1", "true", "yes"}: return True - log.warning("%s has unrecognized value %r, treating as %r.", name, value, default) + _logger.warning("%s has unrecognized value %r, treating as %r.", name, value, default) return default +def _read_win_env_flag(name: str, default: bool) -> bool: + """Read a boolean flag from an environment variable on Windows. + + :return: + On Windows, the flag, or the `default` value if absent or ambiguous. + On all other operating systems, ``False``. + + :note: + This only accesses the environment on Windows. + """ + return sys.platform == "win32" and _read_env_flag(name, default) + + #: We need an easy way to see if Appveyor TCs start failing, #: so the errors marked with this var are considered "acknowledged" ones, awaiting remedy, #: till then, we wish to hide them. @@ -151,8 +163,8 @@ def _read_win_env_flag(name: str, default: bool) -> bool: def unbare_repo(func: Callable[..., T]) -> Callable[..., T]: - """Methods with this decorator raise :class:`.exc.InvalidGitRepositoryError` if they - encounter a bare repository.""" + """Methods with this decorator raise :exc:`~git.exc.InvalidGitRepositoryError` if + they encounter a bare repository.""" from .exc import InvalidGitRepositoryError @@ -200,13 +212,16 @@ def patch_env(name: str, value: str) -> Generator[None, None, None]: def rmtree(path: PathLike) -> None: """Remove the given directory tree recursively. - :note: We use :func:`shutil.rmtree` but adjust its behaviour to see whether files - that couldn't be deleted are read-only. Windows will not remove them in that - case. + :note: + We use :func:`shutil.rmtree` but adjust its behaviour to see whether files that + couldn't be deleted are read-only. Windows will not remove them in that case. """ - def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: - """Callback for :func:`shutil.rmtree`. Works either as ``onexc`` or ``onerror``.""" + def handler(function: Callable[[str], Any], path: str, _excinfo: Any) -> None: + """Callback for :func:`shutil.rmtree`. + + This works as either a ``onexc`` or ``onerror`` style callback. + """ # Is the error an access error? os.chmod(path, stat.S_IWUSR) @@ -219,7 +234,7 @@ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: raise SkipTest(f"FIXME: fails with: PermissionError\n {ex}") from ex raise - if os.name != "nt": + if sys.platform != "win32": shutil.rmtree(path) elif sys.version_info >= (3, 12): shutil.rmtree(path, onexc=handler) @@ -228,18 +243,20 @@ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: def rmfile(path: PathLike) -> None: - """Ensure file deleted also on *Windows* where read-only files need special treatment.""" + """Ensure file deleted also on *Windows* where read-only files need special + treatment.""" if osp.isfile(path): - if os.name == "nt": + if sys.platform == "win32": os.chmod(path, 0o777) os.remove(path) def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * 1024) -> int: - """Copy all data from the source stream into the destination stream in chunks - of size chunk_size. + """Copy all data from the `source` stream into the `destination` stream in chunks + of size `chunk_size`. - :return: Number of bytes written + :return: + Number of bytes written """ br = 0 while True: @@ -253,11 +270,11 @@ def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * def join_path(a: PathLike, *p: PathLike) -> PathLike: - R"""Join path tokens together similar to osp.join, but always use - '/' instead of possibly '\' on Windows.""" - path = str(a) + R"""Join path tokens together similar to osp.join, but always use ``/`` instead of + possibly ``\`` on Windows.""" + path = os.fspath(a) for b in p: - b = str(b) + b = os.fspath(b) if not b: continue if b.startswith("/"): @@ -270,42 +287,105 @@ def join_path(a: PathLike, *p: PathLike) -> PathLike: return path -if os.name == "nt": +if sys.platform == "win32": - def to_native_path_windows(path: PathLike) -> PathLike: - path = str(path) + def to_native_path_windows(path: PathLike) -> str: + path = os.fspath(path) return path.replace("/", "\\") def to_native_path_linux(path: PathLike) -> str: - path = str(path) + path = os.fspath(path) return path.replace("\\", "/") - __all__.append("to_native_path_windows") to_native_path = to_native_path_windows else: # No need for any work on Linux. def to_native_path_linux(path: PathLike) -> str: - return str(path) + return os.fspath(path) to_native_path = to_native_path_linux def join_path_native(a: PathLike, *p: PathLike) -> PathLike: - R"""Like join_path, but makes sure an OS native path is returned. + R"""Like :func:`join_path`, but makes sure an OS native path is returned. This is only needed to play it safe on Windows and to ensure nice paths that only - use '\'. + use ``\``. """ return to_native_path(join_path(a, *p)) +def _is_path_rooted(path: PathLike) -> bool: + r"""Whether ``path`` has a root, including one encoded in a UNC drive. + + On Windows, ``\directory`` is rooted on the current drive without being + absolute, while ``C:\directory`` has both a drive and a root. In contrast, + ``directory`` and the drive-relative ``C:directory`` have no root. + UNC paths are rooted: ``\\server\share`` stores the share in the drive + returned by :func:`os.path.splitdrive`, while ``\\server\share\directory`` + additionally has a rooted tail. + On POSIX, which has no drive concept, this simply distinguishes absolute + paths such as ``/directory`` from relative paths such as ``directory``. + """ + drive, tail = osp.splitdrive(os.fspath(path)) + separators = (os.sep,) if os.altsep is None else (os.sep, os.altsep) + return tail.startswith(separators) or drive.startswith(separators) + + +def _to_relative_path(root: PathLike, path: PathLike) -> str: + r"""Return a normalized Git-style path confined to ``root``. + + A Windows path such as ``\directory`` is rooted but not absolute. Resolve it + against the drive of ``root`` rather than treating it as relative to ``root``. + Drive-relative paths such as ``C:directory`` are rejected because their meaning + depends on process-global per-drive state. + + For example, with ``root`` set to ``C:\repo`` on Windows: + + * ``directory\file`` -> ``directory/file`` + * ``directory\`` -> ``directory/`` + * ``C:\repo\directory\file`` -> ``directory/file`` + * ``\repo\directory\file`` -> ``directory/file`` + * ``C:directory\file`` -> :exc:`ValueError` + * ``C:\other\file`` -> :exc:`ValueError` + + On POSIX, ``/repo/directory/file`` under ``/repo`` similarly becomes + ``directory/file``. A trailing separator is preserved as a Git-style ``/``. + """ + path_str = os.fspath(path) + if not path_str: + return path_str + + drive, _tail = osp.splitdrive(path_str) + rooted = _is_path_rooted(path_str) + if drive and not rooted: + raise ValueError("Drive-relative path %r is not supported" % path_str) + + root_abs = osp.abspath(os.fspath(root)) + path_abs = osp.abspath(osp.join(root_abs, path_str)) + try: + common_path = osp.commonpath([root_abs, path_abs]) + except ValueError as e: + raise ValueError("Path %r is not in repository at %r" % (path_str, root_abs)) from e + if common_path != root_abs: + raise ValueError("Path %r is not in repository at %r" % (path_str, root_abs)) + + relative_path = to_native_path_linux(osp.relpath(path_abs, root_abs)) + separators = (os.sep,) if os.altsep is None else (os.sep, os.altsep) + if path_str.endswith(separators) and relative_path != "." and not relative_path.endswith("/"): + relative_path += "/" + return relative_path + + def assure_directory_exists(path: PathLike, is_file: bool = False) -> bool: """Make sure that the directory pointed to by path exists. - :param is_file: If True, ``path`` is assumed to be a file and handled correctly. + :param is_file: + If ``True``, `path` is assumed to be a file and handled correctly. Otherwise it must be a directory. - :return: True if the directory was created, False if it already existed. + :return: + ``True`` if the directory was created, ``False`` if it already existed. """ if is_file: path = osp.dirname(path) @@ -320,8 +400,8 @@ def _get_exe_extensions() -> Sequence[str]: PATHEXT = os.environ.get("PATHEXT", None) if PATHEXT: return tuple(p.upper() for p in PATHEXT.split(os.pathsep)) - elif os.name == "nt": - return (".BAT", "COM", ".EXE") + elif sys.platform == "win32": + return (".BAT", ".COM", ".EXE") else: return () @@ -333,7 +413,8 @@ def py_where(program: str, path: Optional[PathLike] = None) -> List[str]: :func:`is_cygwin_git`. When a search following all shell rules is needed, :func:`shutil.which` can be used instead. - :note: Neither this function nor :func:`shutil.which` will predict the effect of an + :note: + Neither this function nor :func:`shutil.which` will predict the effect of an executable search on a native Windows system due to a :class:`subprocess.Popen` call without ``shell=True``, because shell and non-shell executable search on Windows differ considerably. @@ -345,13 +426,15 @@ def is_exec(fpath: str) -> bool: return ( osp.isfile(fpath) and os.access(fpath, os.X_OK) - and (os.name != "nt" or not winprog_exts or any(fpath.upper().endswith(ext) for ext in winprog_exts)) + and ( + sys.platform != "win32" or not winprog_exts or any(fpath.upper().endswith(ext) for ext in winprog_exts) + ) ) progs = [] if not path: path = os.environ["PATH"] - for folder in str(path).split(os.pathsep): + for folder in os.fspath(path).split(os.pathsep): folder = folder.strip('"') if folder: exe_path = osp.join(folder, program) @@ -361,13 +444,13 @@ def is_exec(fpath: str) -> bool: return progs -def _cygexpath(drive: Optional[str], path: str) -> str: +def _cygexpath(drive: Optional[str], path: str, expand_vars: bool = True) -> str: if osp.isabs(path) and not drive: # Invoked from `cygpath()` directly with `D:Apps\123`? # It's an error, leave it alone just slashes) p = path # convert to str if AnyPath given else: - p = path and osp.normpath(osp.expandvars(osp.expanduser(path))) + p = path and osp.normpath(osp.expandvars(osp.expanduser(path)) if expand_vars else path) if osp.isabs(p): if drive: # Confusing, maybe a remote system should expand vars. @@ -376,11 +459,11 @@ def _cygexpath(drive: Optional[str], path: str) -> str: p = cygpath(p) elif drive: p = "/proc/cygdrive/%s/%s" % (drive.lower(), p) - p_str = str(p) # ensure it is a str and not AnyPath + p_str = os.fspath(p) # ensure it is a str and not AnyPath return p_str.replace("\\", "/") -_cygpath_parsers: Tuple[Tuple[Pattern[str], Callable, bool], ...] = ( +_cygpath_parsers: Tuple[Tuple[Pattern[str], Callable[..., str], bool], ...] = ( # See: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx # and: https://www.cygwin.com/cygwin-ug-net/using.html#unc-paths ( @@ -395,20 +478,23 @@ def _cygexpath(drive: Optional[str], path: str) -> str: ) -def cygpath(path: str) -> str: +def cygpath(path: str, expand_vars: bool = True) -> str: """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" - path = str(path) # Ensure is str and not AnyPath. + path = os.fspath(path) # Ensure is str and not AnyPath. # Fix to use Paths when 3.5 dropped. Or to be just str if only for URLs? if not path.startswith(("/cygdrive", "//", "/proc/cygdrive")): for regex, parser, recurse in _cygpath_parsers: match = regex.match(path) if match: - path = parser(*match.groups()) + if parser is _cygexpath: + path = parser(*match.groups(), expand_vars=expand_vars) + else: + path = parser(*match.groups()) if recurse: - path = cygpath(path) + path = cygpath(path, expand_vars=expand_vars) break else: - path = _cygexpath(None, path) + path = _cygexpath(None, path, expand_vars=expand_vars) return path @@ -417,7 +503,7 @@ def cygpath(path: str) -> str: def decygpath(path: PathLike) -> str: - path = str(path) + path = os.fspath(path) m = _decygpath_regex.match(path) if m: drive, rest_path = m.groups() @@ -431,25 +517,7 @@ def decygpath(path: PathLike) -> str: _is_cygwin_cache: Dict[str, Optional[bool]] = {} -@overload -def is_cygwin_git(git_executable: None) -> Literal[False]: - ... - - -@overload -def is_cygwin_git(git_executable: PathLike) -> bool: - ... - - -def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: - if os.name == "nt": - # This is Windows-native Python, since Cygwin has os.name == "posix". - return False - - if git_executable is None: - return False - - git_executable = str(git_executable) +def _is_cygwin_git(git_executable: str) -> bool: is_cygwin = _is_cygwin_cache.get(git_executable) # type: Optional[bool] if is_cygwin is None: is_cygwin = False @@ -461,54 +529,80 @@ def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: # Just a name given, not a real path. uname_cmd = osp.join(git_dir, "uname") + + if not (Path(uname_cmd).is_file() and os.access(uname_cmd, os.X_OK)): + _logger.debug(f"Failed checking if running in CYGWIN: {uname_cmd} is not an executable") + _is_cygwin_cache[git_executable] = is_cygwin + return is_cygwin + process = subprocess.Popen([uname_cmd], stdout=subprocess.PIPE, universal_newlines=True) uname_out, _ = process.communicate() # retcode = process.poll() is_cygwin = "CYGWIN" in uname_out except Exception as ex: - log.debug("Failed checking if running in CYGWIN due to: %r", ex) + _logger.debug("Failed checking if running in CYGWIN due to: %r", ex) _is_cygwin_cache[git_executable] = is_cygwin return is_cygwin +@overload +def is_cygwin_git(git_executable: None) -> Literal[False]: ... + + +@overload +def is_cygwin_git(git_executable: PathLike) -> bool: ... + + +def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: + # TODO: when py3.7 support is dropped, use the new interpolation f"{variable=}" + _logger.debug(f"sys.platform={sys.platform!r}, git_executable={git_executable!r}") + if sys.platform != "cygwin": + return False + elif git_executable is None: + return False + else: + return _is_cygwin_git(str(git_executable)) + + def get_user_id() -> str: - """:return: string identifying the currently active system user as name@node""" + """:return: String identifying the currently active system user as ``name@node``""" return "%s@%s" % (getpass.getuser(), platform.node()) -def finalize_process(proc: Union[subprocess.Popen, "Git.AutoInterrupt"], **kwargs: Any) -> None: - """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly""" +def finalize_process(proc: Union["subprocess.Popen[Any]", "Git.AutoInterrupt"], **kwargs: Any) -> None: + """Wait for the process (clone, fetch, pull or push) and handle its errors + accordingly.""" # TODO: No close proc-streams?? proc.wait(**kwargs) @overload -def expand_path(p: None, expand_vars: bool = ...) -> None: - ... +def expand_path(p: None, expand_vars: bool = ...) -> None: ... @overload -def expand_path(p: PathLike, expand_vars: bool = ...) -> str: - # improve these overloads when 3.5 dropped +def expand_path(p: PathLike, expand_vars: bool = ...) -> Optional[PathLike]: + # TODO: Support for Python 3.5 has been dropped, so these overloads can be improved. ... def expand_path(p: Union[None, PathLike], expand_vars: bool = True) -> Optional[PathLike]: - if isinstance(p, pathlib.Path): - return p.resolve() + if p is None: + return None try: - p = osp.expanduser(p) # type: ignore + if isinstance(p, Path): + return p.resolve() + expanded_path = osp.expanduser(os.fspath(p)) if expand_vars: - p = osp.expandvars(p) # type: ignore - return osp.normpath(osp.abspath(p)) # type: ignore + expanded_path = osp.expandvars(expanded_path) + return osp.normpath(osp.abspath(expanded_path)) except Exception: return None def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: - """Parse any command line argument and if one of the elements is an URL with a - username and/or password, replace them by stars (in-place). + """Redact credentials in URLs and HTTP Authorization extra headers in a command line. If nothing is found, this just returns the command line as-is. @@ -518,6 +612,16 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: new_cmdline = [] for index, to_parse in enumerate(cmdline): new_cmdline.append(to_parse) + config_key, separator, header = to_parse.partition("=") + header_name, colon, _ = header.partition(":") + if ( + separator + and colon + and config_key.lower().endswith(".extraheader") + and header_name.strip().lower() == "authorization" + ): + new_cmdline[index] = "%s%s%s%s *****" % (config_key, separator, header_name, colon) + continue try: url = urlsplit(to_parse) # Remove password from the URL if present. @@ -541,10 +645,9 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: class RemoteProgress: - """ - Handler providing an interface to parse progress information emitted by git-push - and git-fetch and to dispatch callbacks allowing subclasses to react to the progress. - """ + """Handler providing an interface to parse progress information emitted by + :manpage:`git-push(1)` and :manpage:`git-fetch(1)` and to dispatch callbacks + allowing subclasses to react to the progress.""" _num_op_codes: int = 9 ( @@ -579,13 +682,17 @@ def __init__(self) -> None: self.error_lines: List[str] = [] self.other_lines: List[str] = [] - def _parse_progress_line(self, line: AnyStr) -> None: - """Parse progress information from the given line as retrieved by git-push - or git-fetch. + def _parse_progress_line(self, line: AnyStr) -> Optional[object]: + """Parse progress information from the given line as retrieved by + :manpage:`git-push(1)` or :manpage:`git-fetch(1)`. - Lines that do not contain progress info are stored in :attr:`other_lines`. - Lines that seem to contain an error (i.e. start with ``error:`` or ``fatal:``) are stored in :attr:`error_lines`. + + The base implementation returns ``None``. Subclasses may return another + value for compatibility with existing overrides, but callers should treat + the return value as unspecified. """ # handle # Counting objects: 4, done. @@ -600,21 +707,7 @@ def _parse_progress_line(self, line: AnyStr) -> None: if self._cur_line.startswith(("error:", "fatal:")): self.error_lines.append(self._cur_line) - return - - # Find escape characters and cut them away - regex will not work with - # them as they are non-ASCII. As git might expect a tty, it will send them. - last_valid_index = None - for i, c in enumerate(reversed(line_str)): - if ord(c) < 32: - # its a slice index - last_valid_index = -i - 1 - # END character was non-ASCII - # END for each character in line - if last_valid_index is not None: - line_str = line_str[:last_valid_index] - # END cut away invalid part - line_str = line_str.rstrip() + return None cur_count, max_count = None, None match = self.re_op_relative.match(line_str) @@ -624,7 +717,7 @@ def _parse_progress_line(self, line: AnyStr) -> None: if not match: self.line_dropped(line_str) self.other_lines.append(line_str) - return + return None # END could not get match op_code = 0 @@ -655,7 +748,7 @@ def _parse_progress_line(self, line: AnyStr) -> None: self.line_dropped(line_str) # Note: Don't add this line to the other lines, as we have to silently # drop it. - return + return None # END handle op code # Figure out stage. @@ -681,16 +774,17 @@ def _parse_progress_line(self, line: AnyStr) -> None: max_count and float(max_count), message, ) + return None def new_message_handler(self) -> Callable[[str], None]: """ :return: - A progress handler suitable for handle_process_output(), passing lines on to - this Progress handler in a suitable format + A progress handler suitable for :func:`~git.cmd.handle_process_output`, + passing lines on to this progress handler in a suitable format. """ def handler(line: AnyStr) -> None: - return self._parse_progress_line(line.rstrip()) + self._parse_progress_line(line.rstrip()) # END handler @@ -712,35 +806,46 @@ def update( :param op_code: Integer allowing to be compared against Operation IDs and stage IDs. - Stage IDs are BEGIN and END. BEGIN will only be set once for each Operation - ID as well as END. It may be that BEGIN and END are set at once in case only - one progress message was emitted due to the speed of the operation. - Between BEGIN and END, none of these flags will be set. + Stage IDs are :const:`BEGIN` and :const:`END`. :const:`BEGIN` will only be + set once for each Operation ID as well as :const:`END`. It may be that + :const:`BEGIN` and :const:`END` are set at once in case only one progress + message was emitted due to the speed of the operation. Between + :const:`BEGIN` and :const:`END`, none of these flags will be set. - Operation IDs are all held within the OP_MASK. Only one Operation ID will - be active per call. + Operation IDs are all held within the :const:`OP_MASK`. Only one Operation + ID will be active per call. - :param cur_count: Current absolute count of items. + :param cur_count: + Current absolute count of items. :param max_count: - The maximum count of items we expect. It may be None in case there is - no maximum number of items or if it is (yet) unknown. + The maximum count of items we expect. It may be ``None`` in case there is no + maximum number of items or if it is (yet) unknown. :param message: - In case of the 'WRITING' operation, it contains the amount of bytes + In case of the :const:`WRITING` operation, it contains the amount of bytes transferred. It may possibly be used for other purposes as well. - You may read the contents of the current line in ``self._cur_line``. + :note: + You may read the contents of the current line in + :attr:`self._cur_line <_cur_line>`. """ pass class CallableRemoteProgress(RemoteProgress): - """An implementation forwarding updates to any callable.""" + """A :class:`RemoteProgress` implementation forwarding updates to any callable. + + :note: + Like direct instances of :class:`RemoteProgress`, instances of this + :class:`CallableRemoteProgress` class are not themselves directly callable. + Rather, instances of this class wrap a callable and forward to it. This should + therefore not be confused with :class:`git.types.CallableProgress`. + """ __slots__ = ("_callable",) - def __init__(self, fn: Callable) -> None: + def __init__(self, fn: Callable[..., Any]) -> None: self._callable = fn super().__init__() @@ -749,13 +854,9 @@ def update(self, *args: Any, **kwargs: Any) -> None: class Actor: - """Actors hold information about a person acting on the repository. They - can be committers and authors or anything with a name and an email as - mentioned in the git log entries.""" - - # PRECOMPILED REGEX - name_only_regex = re.compile(r"<(.*)>") - name_email_regex = re.compile(r"(.*) <(.*?)>") + """Actors hold information about a person acting on the repository. They can be + committers and authors or anything with a name and an email as mentioned in the git + log entries.""" # ENVIRONMENT VARIABLES # These are read when creating new commits. @@ -791,33 +892,31 @@ def __repr__(self) -> str: @classmethod def _from_string(cls, string: str) -> "Actor": - """Create an Actor from a string. + """Create an :class:`Actor` from a string. - :param string: The string, which is expected to be in regular git format:: + :param string: + The string, which is expected to be in regular git format:: - John Doe + John Doe - :return: Actor + :return: + :class:`Actor` """ - m = cls.name_email_regex.search(string) - if m: - name, email = m.groups() - return Actor(name, email) - else: - m = cls.name_only_regex.search(string) - if m: - return Actor(m.group(1), None) - # Assume the best and use the whole string as name. - return Actor(string, None) - # END special case name - # END handle name/email matching + line = string.partition("\n")[0] + left_bracket = line.find("<") + right_bracket = line.find(">", left_bracket + 1) + if left_bracket >= 0 and right_bracket >= 0: + return Actor(line[:left_bracket].rstrip(), line[left_bracket + 1 : right_bracket]) + + # Assume the best and use the whole string as name. + return Actor(string, None) @classmethod def _main_actor( cls, env_name: str, env_email: str, - config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None, + config_reader: Union[None, "GitConfigParser", "SectionConstraint[GitConfigParser]"] = None, ) -> "Actor": actor = Actor("", "") user_id = None # We use this to avoid multiple calls to getpass.getuser(). @@ -853,31 +952,36 @@ def default_name() -> str: return actor @classmethod - def committer(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None) -> "Actor": + def committer( + cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint[GitConfigParser]"] = None + ) -> "Actor": """ - :return: Actor instance corresponding to the configured committer. It behaves - similar to the git implementation, such that the environment will override - configuration values of config_reader. If no value is set at all, it will be - generated. - - :param config_reader: ConfigReader to use to retrieve the values from in case - they are not set in the environment. + :return: + :class:`Actor` instance corresponding to the configured committer. It + behaves similar to the git implementation, such that the environment will + override configuration values of `config_reader`. If no value is set at all, + it will be generated. + + :param config_reader: + ConfigReader to use to retrieve the values from in case they are not set in + the environment. """ return cls._main_actor(cls.env_committer_name, cls.env_committer_email, config_reader) @classmethod - def author(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None) -> "Actor": - """Same as committer(), but defines the main author. It may be specified in the - environment, but defaults to the committer.""" + def author( + cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint[GitConfigParser]"] = None + ) -> "Actor": + """Same as :meth:`committer`, but defines the main author. It may be specified + in the environment, but defaults to the committer.""" return cls._main_actor(cls.env_author_name, cls.env_author_email, config_reader) class Stats: - """ - Represents stat information as presented by git at the end of a merge. It is + """Represents stat information as presented by git at the end of a merge. It is created from the output of a diff operation. - ``Example``:: + Example:: c = Commit( sha1 ) s = c.stats @@ -891,6 +995,7 @@ class Stats: deletions = number of deleted lines as int insertions = number of inserted lines as int lines = total number of lines changed as int, or deletions + insertions + change_type = type of change as str, A|C|D|M|R|T|U|X|B ``full-stat-dict`` @@ -901,15 +1006,17 @@ class Stats: __slots__ = ("total", "files") - def __init__(self, total: Total_TD, files: Dict[PathLike, Files_TD]): + def __init__(self, total: Total_TD, files: Dict[PathLike, Files_TD]) -> None: self.total = total self.files = files @classmethod def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": - """Create a Stat object from output retrieved by git-diff. + """Create a :class:`Stats` object from output retrieved by + :manpage:`git-diff(1)`. - :return: git.Stat + :return: + :class:`git.Stats` """ hsh: HSH_TD = { @@ -917,7 +1024,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": "files": {}, } for line in text.splitlines(): - (raw_insertions, raw_deletions, filename) = line.split("\t") + (change_type, raw_insertions, raw_deletions, filename) = line.split("\t") insertions = raw_insertions != "-" and int(raw_insertions) or 0 deletions = raw_deletions != "-" and int(raw_deletions) or 0 hsh["total"]["insertions"] += insertions @@ -928,28 +1035,30 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": "insertions": insertions, "deletions": deletions, "lines": insertions + deletions, + "change_type": change_type, } hsh["files"][filename.strip()] = files_dict return Stats(hsh["total"], hsh["files"]) class IndexFileSHA1Writer: - """Wrapper around a file-like object that remembers the SHA1 of - the data written to it. It will write a sha when the stream is closed - or if the asked for explicitly using write_sha. + """Wrapper around a file-like object that remembers the SHA1 of the data written to + it. It will write a sha when the stream is closed or if asked for explicitly using + :meth:`write_sha`. Only useful to the index file. - :note: Based on the dulwich project. + :note: + Based on the dulwich project. """ __slots__ = ("f", "sha1") - def __init__(self, f: IO) -> None: + def __init__(self, f: IO[bytes]) -> None: self.f = f self.sha1 = make_sha(b"") - def write(self, data: AnyStr) -> int: + def write(self, data: bytes) -> int: self.sha1.update(data) return self.f.write(data) @@ -991,16 +1100,20 @@ def _lock_file_path(self) -> str: def _has_lock(self) -> bool: """ - :return: True if we have a lock and if the lockfile still exists + :return: + True if we have a lock and if the lockfile still exists - :raise AssertionError: If our lock-file does not exist + :raise AssertionError: + If our lock-file does not exist. """ return self._owns_lock def _obtain_lock_or_raise(self) -> None: - """Create a lock file as flag for other instances, mark our instance as lock-holder. + """Create a lock file as flag for other instances, mark our instance as + lock-holder. - :raise IOError: If a lock was already present or a lock file could not be written + :raise IOError: + If a lock was already present or a lock file could not be written. """ if self._has_lock(): return @@ -1021,7 +1134,9 @@ def _obtain_lock_or_raise(self) -> None: def _obtain_lock(self) -> None: """The default implementation will raise if a lock cannot be obtained. - Subclasses may override this method to provide a different implementation.""" + + Subclasses may override this method to provide a different implementation. + """ return self._obtain_lock_or_raise() def _release_lock(self) -> None: @@ -1029,8 +1144,8 @@ def _release_lock(self) -> None: if not self._has_lock(): return - # If someone removed our file beforehand, lets just flag this issue - # instead of failing, to make it more usable. + # If someone removed our file beforehand, lets just flag this issue instead of + # failing, to make it more usable. lfp = self._lock_file_path() try: rmfile(lfp) @@ -1040,12 +1155,12 @@ def _release_lock(self) -> None: class BlockingLockFile(LockFile): - """The lock file will block until a lock could be obtained, or fail after - a specified timeout. + """The lock file will block until a lock could be obtained, or fail after a + specified timeout. - :note: If the directory containing the lock was removed, an exception will - be raised during the blocking period, preventing hangs as the lock - can never be obtained. + :note: + If the directory containing the lock was removed, an exception will be raised + during the blocking period, preventing hangs as the lock can never be obtained. """ __slots__ = ("_check_interval", "_max_block_time") @@ -1062,15 +1177,16 @@ def __init__( Period of time to sleep until the lock is checked the next time. By default, it waits a nearly unlimited time. - :param max_block_time_s: Maximum amount of seconds we may lock. + :param max_block_time_s: + Maximum amount of seconds we may lock. """ super().__init__(file_path) self._check_interval = check_interval_s self._max_block_time = max_block_time_s def _obtain_lock(self) -> None: - """This method blocks until it obtained the lock, or raises IOError if - it ran out of time or if the parent directory was not available anymore. + """This method blocks until it obtained the lock, or raises :exc:`IOError` if it + ran out of time or if the parent directory was not available anymore. If this method returns, you are guaranteed to own the lock. """ @@ -1104,24 +1220,33 @@ def _obtain_lock(self) -> None: # END endless loop -class IterableList(List[T_IterableObj]): - """ - List of iterable objects allowing to query an object by id or by named index:: +class IterableList(List[T_IterableObj]): # type: ignore[type-var] + """List of iterable objects allowing to query an object by id or by named index:: heads = repo.heads heads.master heads['master'] heads[0] - Iterable parent objects = [Commit, SubModule, Reference, FetchInfo, PushInfo] - Iterable via inheritance = [Head, TagReference, RemoteReference] + Iterable parent objects: + + * :class:`Commit ` + * :class:`Submodule ` + * :class:`Reference ` + * :class:`FetchInfo ` + * :class:`PushInfo ` + + Iterable via inheritance: - It requires an id_attribute name to be set which will be queried from its + * :class:`Head ` + * :class:`TagReference ` + * :class:`RemoteReference ` + + This requires an ``id_attribute`` name to be set which will be queried from its contained items to have a means for comparison. - A prefix can be specified which is to be used in case the id returned by the - items always contains a prefix that does not matter to the user, so it - can be left out. + A prefix can be specified which is to be used in case the id returned by the items + always contains a prefix that does not matter to the user, so it can be left out. """ __slots__ = ("_id_attr", "_prefix") @@ -1130,6 +1255,7 @@ def __new__(cls, id_attr: str, prefix: str = "") -> "IterableList[T_IterableObj] return super().__new__(cls) def __init__(self, id_attr: str, prefix: str = "") -> None: + super().__init__() self._id_attr = id_attr self._prefix = prefix @@ -1159,25 +1285,23 @@ def __getattr__(self, attr: str) -> T_IterableObj: # END for each item return list.__getattribute__(self, attr) - def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_IterableObj: # type: ignore - assert isinstance(index, (int, str, slice)), "Index of IterableList should be an int or str" - + def __getitem__( # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] + self, index: Union[SupportsIndex, int, slice, str] + ) -> T_IterableObj: if isinstance(index, int): return list.__getitem__(self, index) elif isinstance(index, slice): raise ValueError("Index should be an int or str") else: try: - return getattr(self, index) + return getattr(self, cast(str, index)) except AttributeError as e: - raise IndexError("No item found with id %r" % (self._prefix + index)) from e + raise IndexError(f"No item found with id {self._prefix}{index}") from e # END handle getattr def __delitem__(self, index: Union[SupportsIndex, int, slice, str]) -> None: - assert isinstance(index, (int, str)), "Index of IterableList should be an int or str" - delindex = cast(int, index) - if not isinstance(index, int): + if isinstance(index, str): delindex = -1 name = self._prefix + index for i, item in enumerate(self): @@ -1198,7 +1322,14 @@ class IterableObj(Protocol): """Defines an interface for iterable items, so there is a uniform way to retrieve and iterate items within the git repository. - Subclasses = [Submodule, Commit, Reference, PushInfo, FetchInfo, Remote] + Subclasses: + + * :class:`Submodule ` + * :class:`Commit ` + * :class:`Reference ` + * :class:`PushInfo ` + * :class:`FetchInfo ` + * :class:`Remote ` """ __slots__ = () @@ -1211,11 +1342,12 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator[T_Itera # Return-typed to be compatible with subtypes e.g. Remote. """Find (all) items of this type. - Subclasses can specify ``args`` and ``kwargs`` differently, and may use them for + Subclasses can specify `args` and `kwargs` differently, and may use them for filtering. However, when the method is called with no additional positional or keyword arguments, subclasses are obliged to to yield all items. - :return: Iterator yielding Items + :return: + Iterator yielding Items """ raise NotImplementedError("To be implemented by Subclass") @@ -1225,22 +1357,25 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_I For more information about the arguments, see :meth:`iter_items`. - :note: Favor the :meth:`iter_items` method as it will avoid eagerly collecting - all items. When there are many items, that can slow performance and increase + :note: + Favor the :meth:`iter_items` method as it will avoid eagerly collecting all + items. When there are many items, that can slow performance and increase memory usage. - :return: list(Item,...) list of item instances + :return: + list(Item,...) list of item instances """ - out_list: IterableList = IterableList(cls._id_attribute_) + out_list: IterableList[T_IterableObj] = IterableList(cls._id_attribute_) out_list.extend(cls.iter_items(repo, *args, **kwargs)) return out_list class IterableClassWatcher(type): - """Metaclass that issues :class:`DeprecationWarning` when :class:`git.util.Iterable` + """Metaclass that issues :exc:`DeprecationWarning` when :class:`git.util.Iterable` is subclassed.""" - def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: + def __init__(cls, name: str, bases: Tuple[type, ...], clsdict: Dict[str, Any]) -> None: + super().__init__(name, bases, clsdict) for base in bases: if type(base) is IterableClassWatcher: warnings.warn( @@ -1272,7 +1407,8 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: See :meth:`IterableObj.iter_items` for details on usage. - :return: Iterator yielding Items + :return: + Iterator yielding Items """ raise NotImplementedError("To be implemented by Subclass") @@ -1284,7 +1420,8 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: See :meth:`IterableObj.list_items` for details on usage. - :return: list(Item,...) list of item instances + :return: + list(Item,...) list of item instances """ out_list: Any = IterableList(cls._id_attribute_) out_list.extend(cls.iter_items(repo, *args, **kwargs)) @@ -1292,8 +1429,3 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: # } END classes - - -class NullHandler(logging.Handler): - def emit(self, record: object) -> None: - pass diff --git a/gitdb/.coveragerc b/gitdb/.coveragerc new file mode 100644 index 000000000..71b6ef701 --- /dev/null +++ b/gitdb/.coveragerc @@ -0,0 +1,7 @@ +[run] +source = gitdb + +; to make nosetests happy +[report] +include = */gitdb/* +omit = */gitdb/ext/* diff --git a/gitdb/.gitignore b/gitdb/.gitignore new file mode 100644 index 000000000..8b7da9277 --- /dev/null +++ b/gitdb/.gitignore @@ -0,0 +1,10 @@ +MANIFEST +.coverage +build/ +dist/ +*.pyc +*.o +*.so +.noseids +*.sublime-workspace +*.egg-info diff --git a/gitdb/AUTHORS b/gitdb/AUTHORS new file mode 100644 index 000000000..6c7e9b997 --- /dev/null +++ b/gitdb/AUTHORS @@ -0,0 +1,4 @@ +Creator: Sebastian Thiel + +Contributors: + - Ram Rachum (@cool-RR) diff --git a/gitdb/LICENSE b/gitdb/LICENSE new file mode 100644 index 000000000..0d6fe8bdb --- /dev/null +++ b/gitdb/LICENSE @@ -0,0 +1,42 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the GitDB project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Additional Licenses +------------------- +The files at +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx +and +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack +are licensed under GNU GPL as part of the git source repository, +see http://en.wikipedia.org/wiki/Git_%28software%29 for more information. + +They are not required for the actual operation, which is why they are not found +in the distribution package. diff --git a/gitdb/MANIFEST.in b/gitdb/MANIFEST.in new file mode 100644 index 000000000..b939b5ded --- /dev/null +++ b/gitdb/MANIFEST.in @@ -0,0 +1,18 @@ +include VERSION +include LICENSE +include CHANGES +include AUTHORS +include README +include MANIFEST.in + +include gitdb/_fun.c +include gitdb/_delta_apply.c +include gitdb/_delta_apply.h + +graft gitdb/test + +global-exclude .git* +global-exclude *.pyc +global-exclude *.so +global-exclude *.dll +global-exclude *.o diff --git a/gitdb/Makefile b/gitdb/Makefile new file mode 100644 index 000000000..ad0912ffe --- /dev/null +++ b/gitdb/Makefile @@ -0,0 +1,7 @@ +.PHONY: all clean + +all: + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all + +clean: + rm -rf build/ dist/ .eggs/ .tox/ diff --git a/gitdb/README.rst b/gitdb/README.rst new file mode 100644 index 000000000..45344a078 --- /dev/null +++ b/gitdb/README.rst @@ -0,0 +1,69 @@ +GitDB +===== + +GitDB allows you to access bare git repositories for reading and writing. It aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to handle large objects with a small memory footprint. + +Installation +============ + +.. image:: https://img.shields.io/pypi/v/gitdb.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Latest Version +.. image:: https://img.shields.io/pypi/pyversions/gitdb.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Supported Python versions +.. image:: https://readthedocs.org/projects/gitdb/badge/?version=latest + :target: https://readthedocs.org/projects/gitdb/?badge=latest + :alt: Documentation Status + +From `PyPI `_:: + + pip install gitdb + +SPEEDUPS +======== + +If you want to go up to 20% faster, you can install gitdb-speedups with:: + + pip install gitdb-speedups + +However, please note that gitdb-speedups is not currently maintained. + +REQUIREMENTS +============ + +* smmap - declared as a dependency, automatically installed +* pytest - for running the tests + +SOURCE +====== + +The source is available in the GitPython repository on GitHub: + +https://github.com/gitpython-developers/GitPython/tree/main/gitdb + +Run the tests with:: + + pytest + +DEVELOPMENT +=========== + +.. image:: https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml/badge.svg + :target: https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml + +The library is considered mature, and not under active development. Its primary (known) use is in GitPython. + +INFRASTRUCTURE +============== + +* Discussions + * https://github.com/gitpython-developers/GitPython/discussions + +* Issue Tracker + * https://github.com/gitpython-developers/GitPython/issues + +LICENSE +======= + +New BSD License diff --git a/gitdb/SECURITY.md b/gitdb/SECURITY.md new file mode 100644 index 000000000..95389ff6e --- /dev/null +++ b/gitdb/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See [GitPython](https://github.com/gitpython-developers/GitPython/blob/main/SECURITY.md). Vulnerabilities found in `gitdb` can be reported there. diff --git a/gitdb/doc/.gitignore b/gitdb/doc/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/gitdb/doc/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/gitdb/doc/Makefile b/gitdb/doc/Makefile new file mode 100644 index 000000000..b10926ae2 --- /dev/null +++ b/gitdb/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/GitDB.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/GitDB.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/gitdb/doc/source/algorithm.rst b/gitdb/doc/source/algorithm.rst new file mode 100644 index 000000000..2e01b3fbe --- /dev/null +++ b/gitdb/doc/source/algorithm.rst @@ -0,0 +1,97 @@ +######################## +Discussion of Algorithms +######################## + +************ +Introduction +************ +As you know, the pure-python object database support for GitPython is provided by the GitDB project. It is meant to be my backup plan to ensure that the DataVault (http://www.youtube.com/user/ByronBates99?feature=mhum#p/c/2A5C6EF5BDA8DB5C ) can handle reading huge files, especially those which were consolidated into packs. A nearly fully packed state is anticipated for the data-vaults repository, and reading these packs efficiently is an essential task. + +This document informs you about my findings in the struggle to improve the way packs are read to reduce memory usage required to handle huge files. It will try to conclude where future development could go to assure big delta-packed files can be read without the need of 8GB+ RAM. + +GitDB's main feature is the use of streams, hence the amount of memory used to read a database object is minimized, at the cost of some additional processing overhead to keep track of the stream state. This works great for legacy objects, which are essentially a zip-compressed byte-stream. + +Streaming data from delta-packed objects is far more difficult though, and only technically possible within certain limits, and at relatively high processing costs. My first observation was that there doesn't appear to be 'the one and only' algorithm which is generally superior. They all have their pros and cons, but fortunately this allows the implementation to choose the one most suited based on the amount of delta streams, as well as the size of the base, which allows an early and cheap estimate of the target size of the final data. + +********************************** +Traditional Delta-Apply-Algorithms +********************************** + +The brute-force CGit delta-apply algorithm +========================================== +CGit employs a simple and relatively brute-force algorithm, which resolves all delta streams recursively. When the recursion reaches the base level of the deltas, it will be decompressed into a buffer, then the first delta gets decompressed into a second buffer. From that, the target size of the delta can be extracted, to allocated a third buffer to hold the result of the operation, which consists of reading the delta stream byte-wise, to apply the operations in order, as described by single-byte opcodes. During recursion, each target buffer of the preceding delta-apply operation is used as base buffer for the next delta-apply operation, until the last delta was applied, leaving the final target buffer as result. + +About Delta-Opcodes +------------------- +There are only two kinds of opcodes, 'add-bytes' and 'copy-from-base'. One 'add-bytes' opcode can encode up to 7 bit of additional bytes to be copied from the delta stream into the target buffer. +A 'copy-from-base' opcode encodes a 32 bit offset into the base buffer, as well as the amount of bytes to be copied, which are up to 2^24 bytes. We may conclude that delta-bases may not be larger than 2^32+2^24 bytes in the current, extensible, implementation. +When generating the delta, git prefers copy operations over add operations, as they are much more efficient. Usually, the most recent, or biggest version of a file is used as base, whereas older and smaller versions of the file are expressed by copying only portions of the newest file. As it is not efficiently possible to represent all changes that way, add-bytes operations fill the gap where needed. All this explains why git can only add 128 bytes with one opcode, as it tries to minimize their use. +This implies that recent file history can usually be extracted faster than old history, which may involve many more deltas. + +Performance considerations +-------------------------- +The performance bottleneck of this algorithm appear to be the throughput of your RAM, as both opcodes will just trigger memcpy operations from one memory location to another, times the amount of deltas to apply. This in fact is very fast, even for big files above 100 MB. +Memory allocation could become an issue as you need the base buffer, the target buffer as well as the decompressed delta stream in memory at the same time. The continuous allocation and deallocation of possibly big buffers may support memory fragmentation. Whether it really kicks in, especially on 64 bit machines, is unproven though. +Nonetheless, the cgit implementation is currently the fastest one. + +The brute-force GitDB algorithm +=============================== +Despite of working essentially the same way as the CGit brute-force algorithm, GitDB minimizes the amount of allocations to 2 + num of deltas. The amount memory allocated peaks whiles the deltas are applied, as the base and target buffer, as well as the decompressed stream, are held in memory. +To achieve this, GitDB retrieves all delta-streams in advance, and peaks into their header information to determine the maximum size of the target buffer, just by reading 512 bytes of the compressed stream. If there is more than one delta to apply, the base buffer is set large enough to hold the biggest target buffer required by the delta streams. +Now it is possible to iterate all deltas, oldest first, newest last, and apply them using the buffers. At the end of each iteration, the buffers are swapped. + +Performance Results +------------------- +The performance test is performed on an aggressively packed repository with the history of cgit. 5000 sha's are extracted and read one after another. The delta-chains have a length of 0 to about 35. The pure-python implementation can stream the data of all objects (totaling 62,2 MiB) with an average rate of 8.1 MiB/s, which equals about 654 streams/s. +There are two bottlenecks: The major is the collection of the delta streams, which involves plenty of pack-lookup. This lookup is expensive in python, and is overly expensive. Its not overly critical though, as it only limits the amount of streams per second, not the actual data rate when applying the deltas. +Applying the deltas happens to be the second bottleneck, if the files to be processed get bigger. The more opcodes have to be processed, the more python slow function calls will dominate the result. As an example, it takes nearly 8 seconds to unpack a 125 MB file, where cgit only takes 2.4 s. + +To eliminate a few performance concerns, some key routines were rewritten in C. This changes the numbers of this particular test significantly, but not drastically, as the major bottleneck (delta collection) is still in place. Another performance reduction is due to the fact that plenty of other code related to the deltas is still pure-python. +Now all 5000 objects can be read at a rate of 11.1 MiB /s, or 892 streams/s. Fortunately, unpacking a big object is now done in 2.5s, which is just a tad slower than cgit, but with possibly less memory fragmentation. + + +************************************** +Paving the way towards delta streaming +************************************** + +GitDB's reverse delta aggregation algorithm +=========================================== +The idea of this algorithm is to merge all delta streams into one, which can then be applied in just one go. + +In the current implementation, delta streams are parsed into DeltaChunks (->**DC**). Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. +Add-bytes DCs additional store their data to apply, copy-from-base DCs store the offset into the base buffer from which to copy bytes. + +During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream.. + +The merging works by following a set of rules: + * Merge into the top-level delta from the youngest ancestor delta to the oldest one + * When merging one ADS, iterate from the first to the last chunk in TDS, then: + + * skip all add-bytes DCs. If bytes are added, these will always overwrite any operation coming from any ADS at the same offset. + * copy-from-base DCs will copy a slice of the respective portion of the ADS ( as defined by their base offset ) and use it to replace the original chunk. This acts as a 'virtual' copy-from-base operation. + + * Finish the merge once all ADS have been handled, or once the TDS only consists of add-byte DCs. The remaining copy-from-base DCs will copy from the original base buffer accordingly. + +Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. Here it is already possible to stream the result, which is feasible only if the memory of the base buffer + the memory of the TDS are smaller than a full size target buffer. Streaming will always make sense if the peak resulting from having the base, target and TDS buffers in memory together is unaffordable. + +The memory consumption during the TDS processing is only the condensed delta-bytes, for each ADS an additional index is required which costs 8 byte per DC. When applying the TDS, one requires an allocated base buffer too.The target buffer can be allocated, but may be a writer as well. + +Performance Results +------------------- +The benchmarking context was the same as for the brute-force GitDB algorithm. This implementation is far more complex than the said brute-force implementation, which clearly reflects in the numbers. It's pure-python throughput is at only 1.1 MiB/s, which equals 89 streams/s. +The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. + +To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. +The throughput reaches 15.2 MiB/s, which equals 1221 streams/s, which makes it nearly 14 times faster than the pure python version, and amazingly even 1.35 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. + +A 125 MB file took 2.5 seconds to unpack for instance, which is only 20% slower than the c implementation of the brute-force algorithm. + + +Future work +=========== + +Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. + +The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be paid with extra pre-processing time. + +A very first and simple implementation could avoid memory peaks by streaming the TDS in conjunction with a base buffer, instead of writing everything into a fully allocated target buffer. diff --git a/gitdb/doc/source/api.rst b/gitdb/doc/source/api.rst new file mode 100644 index 000000000..9dea95764 --- /dev/null +++ b/gitdb/doc/source/api.rst @@ -0,0 +1,113 @@ +.. _api-label: + +############# +API Reference +############# + +**************** +Database.Base +**************** + +.. automodule:: gitdb.db.base + :members: + :undoc-members: + +**************** +Database.Git +**************** + +.. automodule:: gitdb.db.git + :members: + :undoc-members: + +**************** +Database.Loose +**************** + +.. automodule:: gitdb.db.loose + :members: + :undoc-members: + +**************** +Database.Memory +**************** + +.. automodule:: gitdb.db.mem + :members: + :undoc-members: + +**************** +Database.Pack +**************** + +.. automodule:: gitdb.db.pack + :members: + :undoc-members: + +****************** +Database.Reference +****************** + +.. automodule:: gitdb.db.ref + :members: + :undoc-members: + +************ +Base +************ + +.. automodule:: gitdb.base + :members: + :undoc-members: + +************ +Functions +************ + +.. automodule:: gitdb.fun + :members: + :undoc-members: + +************ +Pack +************ + +.. automodule:: gitdb.pack + :members: + :undoc-members: + +************ +Streams +************ + +.. automodule:: gitdb.stream + :members: + :undoc-members: + +************ +Types +************ + +.. automodule:: gitdb.typ + :members: + :undoc-members: + + +************ +Utilities +************ + +.. automodule:: gitdb.util + :members: + :undoc-members: + + + + + + + + + + + diff --git a/gitdb/doc/source/changes.rst b/gitdb/doc/source/changes.rst new file mode 100644 index 000000000..b4340e4dc --- /dev/null +++ b/gitdb/doc/source/changes.rst @@ -0,0 +1,141 @@ +######### +Changelog +######### + +****** +4.0.12 +****** + +- various improvements - please see the release on GitHub for details. + +****** +4.0.11 +****** + +- various improvements - please see the release on GitHub for details. + +****** +4.0.10 +****** + +- improvements to the way external packages are imported. + +***** +4.0.9 +***** + +- re-release of 4.0.8 to get a valid signature. + +***** +4.0.8 +***** + +* drop support for python 3.4 and 3.5 due to EOL +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + +***** +4.0.7 +***** + +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + +***** +4.0.6 +***** + +* Bumped upper bound for smmap requirement + (`#67 `_, + `#68 `_) + +***** +4.0.5 +***** + +* Re-release of 4.0.4, with known signature + +***** +4.0.4 +***** + +* Support for PyOxidizer + +***** +4.0.2 +***** + +* Updated to release as Pure Python Wheel rather than Universal Wheel + (`#62 `_) + +***** +4.0.1 +***** + +* Switched back to the gitdb package name on PyPI and fixed the gitdb2 mirror package + (`#59 `_) +* Switched back to require smmap package and fixed version requirement to >= 3.0.1, < 4 + (`#59 `_) +* Updated smmap submodule + +*********** +3.0.3.post1 +*********** + +* Fixed changelogs for v3.0.2 and v3.0.3 + +***** +3.0.3 +***** + +* Changed ``force_bytes`` to use UTF-8 encoding by default + (`#49 `_) +* Restricted smmap2 version requirement to < 3 +* Updated requirements.txt + +***** +3.0.2 +***** + +* Removed Python 2 compatibility shims + (`#56 `_) + +***** +0.6.1 +***** + +* Fixed possibly critical error, see https://github.com/gitpython-developers/GitPython/issues/220 + + - However, it only seems to occur on high-entropy data and didn't reoccour after the fix + +***** +0.6.0 +***** + +* Added support got python 3.X +* Removed all `async` dependencies and all `*_async` versions of methods with it. + +***** +0.5.4 +***** +* Adjusted implementation to use the SlidingMemoryManager by default in python 2.6 for efficiency reasons. In Python 2.4, the StaticMemoryManager will be used instead. + +***** +0.5.3 +***** +* Added support for smmap. SmartMMap allows resources to be managed and controlled. This brings the implementation closer to the way git handles memory maps, such that unused cached memory maps will automatically be freed once a resource limit is hit. The memory limit on 32 bit systems remains though as a sliding mmap implementation is not used for performance reasons. + +***** +0.5.2 +***** +* Improved performance of the c implementation, which now uses reverse-delta-aggregation to make a memory bound operation CPU bound. + +***** +0.5.1 +***** +* Restored most basic python 2.4 compatibility, such that gitdb can be imported within python 2.4, pack access cannot work though. This at least allows Super-Projects to provide their own workarounds, or use everything but pack support. + +***** +0.5.0 +***** +Initial Release diff --git a/gitdb/doc/source/conf.py b/gitdb/doc/source/conf.py new file mode 100644 index 000000000..b387f6094 --- /dev/null +++ b/gitdb/doc/source/conf.py @@ -0,0 +1,193 @@ +# +# GitDB documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 30 00:01:32 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.append(os.path.abspath('../../')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'GitDB' +copyright = '2011, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.5' +# The full version, including alpha/beta/rc tags. +release = '0.5.3' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +html_theme_options = { + "stickysidebar": "true" +} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'GitDBdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'GitDB.tex', 'GitDB Documentation', + 'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/gitdb/doc/source/index.rst b/gitdb/doc/source/index.rst new file mode 100644 index 000000000..5223e6bd9 --- /dev/null +++ b/gitdb/doc/source/index.rst @@ -0,0 +1,26 @@ +.. GitDB documentation master file, created by + sphinx-quickstart on Wed Jun 30 00:01:32 2010. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to GitDB's documentation! +================================= + +Contents: + +.. toctree:: + :maxdepth: 2 + + intro + tutorial + api + algorithm + changes + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/gitdb/doc/source/intro.rst b/gitdb/doc/source/intro.rst new file mode 100644 index 000000000..434138616 --- /dev/null +++ b/gitdb/doc/source/intro.rst @@ -0,0 +1,38 @@ +######## +Overview +######## + +The *GitDB* project implements interfaces to allow read and write access to git repositories. In its core lies the *db* package, which contains all database types necessary to read a complete git repository. These are the ``LooseObjectDB``, the ``PackedDB`` and the ``ReferenceDB`` which are combined into the ``GitDB`` to combine every aspect of the git database. + +For this to work, GitDB implements pack reading, as well as loose object reading and writing. Data is always encapsulated in streams, which allows huge files to be handled as well as small ones, usually only chunks of the stream are kept in memory for processing, never the whole stream at once. + +Interfaces are used to describe the API, making it easy to provide alternate implementations. + +================ +Installing GitDB +================ +Its easiest to install gitdb using the *pip* program:: + + $ pip install gitdb + +As the command will install gitdb in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +=============== +Getting Started +=============== +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +================= +Source Repository +================= +The latest source can be cloned using git from github: + + * https://github.com/gitpython-developers/gitdb + +License Information +=================== +*GitDB* is licensed under the New BSD License. diff --git a/gitdb/doc/source/tutorial.rst b/gitdb/doc/source/tutorial.rst new file mode 100644 index 000000000..55a737f6b --- /dev/null +++ b/gitdb/doc/source/tutorial.rst @@ -0,0 +1,114 @@ +.. _tutorial-label: + +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying types. + +****** +Design +****** +The *GitDB* project models a standard git object database and implements it in pure python. This means that data, being classified by one of four types, can can be stored in the database and will in future be referred to by the generated SHA1 key, which is a 20 byte string within python. + +*GitDB* implements *RW* access to loose objects, as well as *RO* access to packed objects. Compound Databases allow to combine multiple object databases into one. + +All data is read and written using streams, which effectively prevents more than a chunk of the data being kept in memory at once mostly [#]_. + +******* +Streams +******* +In order to assure the object database can handle objects of any size, a stream interface is used for data retrieval as well as to fill data into the database. + +Basic Stream Types +================== +There are two fundamentally different types of streams, **IStream**\ s and **OStream**\ s. IStreams are mutable and are used to provide data streams to the database to create new objects. + +OStreams are immutable and are used to read data from the database. The base of this type, **OInfo**, contains only type and size information of the queried object, but no stream, which is slightly faster to retrieve depending on the database. + +OStreams are tuples, IStreams are lists. Both, OInfo and OStream, have the same member ordering which allows quick conversion from one type to another. + +**************************** +Data Query and Data Addition +**************************** +Databases support query and/or addition of objects using simple interfaces. They are called **ObjectDBR** for read-only access, and **ObjectDBW** for write access to create new objects. + +Both have two sets of methods, one of which allows interacting with single objects, the other one allowing to handle a stream of objects simultaneously and asynchronously. + +Acquiring information about an object from a database is easy if you have a SHA1 to refer to the object:: + + + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + # END for each sha in database + +To store information, you prepare an *IStream* object with the required information. The provided stream will be read and converted into an object, and the respective 20 byte SHA1 identifier is stored in the IStream object:: + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + +********************** +Asynchronous Operation +********************** +For each read or write method that allows a single-object to be handled, an *_async* version exists which reads items to be processed from a channel, and writes the operation's result into an output channel that is read by the caller or by other async methods, to support chaining. + +Using asynchronous operations is easy, but chaining multiple operations together to form a complex one would require you to read the docs of the *async* package. At the current time, due to the *GIL*, the *GitDB* can only achieve true concurrency during zlib compression and decompression if big objects, if the respective c modules where compiled in *async*. + +Asynchronous operations are scheduled by a *ThreadPool* which resides in the *gitdb.util* module:: + + from gitdb.util import pool + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) + + +Use async methods with readers, which supply items to be processed. The result is given through readers as well:: + + from async import IteratorReader + + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + + +********* +Databases +********* +A database implements different interfaces, one if which will always be the *ObjectDBR* interface to support reading of object information and streams. + +The *Loose Object Database* as well as the *Packed Object Database* are *File Databases*, hence they operate on a directory which contains files they can read. + +File databases implementing the *ObjectDBW* interface can also be forced to write their output into the specified stream, using the ``set_ostream`` method. This effectively allows you to redirect its output to anywhere you like. + +*Compound Databases* are not implementing their own access type, but instead combine multiple database implementations into one. Examples for this database type are the *Reference Database*, which reads object locations from a file, and the *GitDB* which combines loose, packed and referenced objects into one database interface. + +For more information about the individual database types, please see the :ref:`API Reference `, and the unittests for the respective types. + + +---- + +.. [#] When reading streams from packs, all deltas are currently applied and the result written into a memory map before the first byte is returned. Future versions of the delta-apply algorithm might improve on this. diff --git a/gitdb/gitdb.pro b/gitdb/gitdb.pro new file mode 100644 index 000000000..a682615c4 --- /dev/null +++ b/gitdb/gitdb.pro @@ -0,0 +1,48 @@ + +OTHER_FILES += \ + setup.py \ + README.rst \ + MANIFEST \ + Makefile \ + LICENSE \ + AUTHORS \ + gitdb/util.py \ + gitdb/typ.py \ + gitdb/stream.py \ + gitdb/pack.py \ + gitdb/__init__.py \ + gitdb/fun.py \ + gitdb/exc.py \ + gitdb/base.py \ + doc/source/tutorial.rst \ + doc/source/intro.rst \ + doc/source/index.rst \ + doc/source/conf.py \ + doc/source/changes.rst \ + doc/source/api.rst \ + doc/source/algorithm.rst \ + gitdb/db/ref.py \ + gitdb/db/pack.py \ + gitdb/db/mem.py \ + gitdb/db/loose.py \ + gitdb/db/__init__.py \ + gitdb/db/git.py \ + gitdb/db/base.py \ + gitdb/test/test_util.py \ + gitdb/test/test_stream.py \ + gitdb/test/test_pack.py \ + gitdb/test/test_example.py \ + gitdb/test/test_base.py \ + gitdb/test/lib.py \ + gitdb/test/__init__.py \ + gitdb/test/performance/test_stream.py \ + gitdb/test/performance/test_pack_streaming.py \ + gitdb/test/performance/test_pack.py \ + gitdb/test/performance/lib.py + +HEADERS += \ + gitdb/_delta_apply.h + +SOURCES += \ + gitdb/_fun.c \ + gitdb/_delta_apply.c diff --git a/gitdb/gitdb.pro.user b/gitdb/gitdb.pro.user new file mode 100644 index 000000000..3ca1e2182 --- /dev/null +++ b/gitdb/gitdb.pro.user @@ -0,0 +1,266 @@ + + + + ProjectExplorer.Project.ActiveTarget + 0 + + + ProjectExplorer.Project.EditorSettings + + Default + + + + ProjectExplorer.Project.Target.0 + + Desktop + + Qt4ProjectManager.Target.DesktopTarget + 0 + 0 + 1 + + + 0 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + false + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + 4 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + No deployment + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + gitdb + + Qt4ProjectManager.Qt4RunConfiguration + 2 + + gitdb.pro + false + false + + false + + 3768 + true + false + + + + /usr/bin/pytest + gitdb/test/test_pack.py + + 2 + python + false + + $BUILDDIR + Run python + test-pack + ProjectExplorer.CustomExecutableRunConfiguration + 3768 + true + false + + 2 + + + + ProjectExplorer.Project.TargetCount + 1 + + + ProjectExplorer.Project.Updater.EnvironmentId + {d38778e3-6b24-4419-8fc3-c8cc320f55e0} + + + ProjectExplorer.Project.Updater.FileVersion + 8 + + diff --git a/gitdb/gitdb/__init__.py b/gitdb/gitdb/__init__.py new file mode 100644 index 000000000..04920428d --- /dev/null +++ b/gitdb/gitdb/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Initialize the object database module""" + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/gitdb" +version_info = (4, 0, 12) +__version__ = '.'.join(str(i) for i in version_info) + +# default imports +from gitdb.base import * +from gitdb.db import * +from gitdb.stream import * diff --git a/gitdb/gitdb/base.py b/gitdb/gitdb/base.py new file mode 100644 index 000000000..9a23a4f71 --- /dev/null +++ b/gitdb/gitdb/base.py @@ -0,0 +1,315 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module with basic data structures - they are designed to be lightweight and fast""" +from gitdb.util import bin_to_hex + +from gitdb.fun import ( + type_id_to_type_map, + type_to_type_id_map +) + +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream') + +#{ ODB Bases + + +class OInfo(tuple): + + """Carries information about an object in an ODB, providing information + about the binary sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.binsha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lightweight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return self[0] + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + @property + def type(self): + return self[1] + + @property + def type_id(self): + return type_to_type_id_map[self[1]] + + @property + def size(self): + return self[2] + #} END interface + + +class OPackInfo(tuple): + + """As OInfo, but provides a type_id property to retrieve the numerical type id, and + does not include a sha. + + Additionally, the pack_offset is the absolute offset into the packfile at which + all object information is located. The data_offset property points to the absolute + location in the pack at which that actual data stream can be found.""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size): + return tuple.__new__(cls, (packoffset, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + + @property + def pack_offset(self): + return self[0] + + @property + def type(self): + return type_id_to_type_map[self[1]] + + @property + def type_id(self): + return self[1] + + @property + def size(self): + return self[2] + + #} END interface + + +class ODeltaPackInfo(OPackInfo): + + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the negative offset from the pack_offset, so that pack_offset - delta_info yields + the pack offset of the base object""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[3] + #} END interface + + +class OStream(OInfo): + + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + + #} END stream reader interface + + +class ODeltaStream(OStream): + + """Uses size info of its stream, delaying reads""" + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + + @property + def size(self): + return self[3].size + + #} END stream reader interface + + +class OPackStream(OPackInfo): + + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (packoffset, type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class ODeltaPackStream(ODeltaPackInfo): + + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[4].read(size) + + @property + def stream(self): + return self[4] + #} END stream reader interface + + +class IStream(list): + + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_binsha(self, binsha): + self[0] = binsha + + def _binsha(self): + return self[0] + + binsha = property(_binsha, _set_binsha) + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + + +class InvalidOInfo(tuple): + + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def binsha(self): + return self[0] + + @property + def hexsha(self): + return bin_to_hex(self[0]) + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] + + +class InvalidOStream(InvalidOInfo): + + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + +#} END ODB Bases diff --git a/gitdb/gitdb/const.py b/gitdb/gitdb/const.py new file mode 100644 index 000000000..6391d796f --- /dev/null +++ b/gitdb/gitdb/const.py @@ -0,0 +1,4 @@ +BYTE_SPACE = b' ' +NULL_BYTE = b'\0' +NULL_HEX_SHA = "0" * 40 +NULL_BIN_SHA = NULL_BYTE * 20 diff --git a/gitdb/gitdb/db/__init__.py b/gitdb/gitdb/db/__init__.py new file mode 100644 index 000000000..20fd2280d --- /dev/null +++ b/gitdb/gitdb/db/__init__.py @@ -0,0 +1,11 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ + +from gitdb.db.base import * +from gitdb.db.loose import * +from gitdb.db.mem import * +from gitdb.db.pack import * +from gitdb.db.git import * +from gitdb.db.ref import * diff --git a/gitdb/gitdb/db/base.py b/gitdb/gitdb/db/base.py new file mode 100644 index 000000000..7312fe00f --- /dev/null +++ b/gitdb/gitdb/db/base.py @@ -0,0 +1,278 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains implementations of database retrieveing objects""" +from gitdb.util import ( + join, + LazyMixin, + hex_to_bin +) + +from gitdb.utils.encoding import force_text +from gitdb.exc import ( + BadObject, + AmbiguousObjectName +) + +from itertools import chain +from functools import reduce + + +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') + + +class ObjectDBR: + + """Defines an interface for object database lookup. + Objects are identified either by their 20 byte bin sha""" + + def __contains__(self, sha): + return self.has_obj + + #{ Query Interface + def has_object(self, sha): + """ + Whether the object identified by the given 20 bytes + binary sha is contained in the database + + :return: True if the object identified by the given 20 bytes + binary sha is contained in the database""" + raise NotImplementedError("To be implemented in subclass") + + def info(self, sha): + """ :return: OInfo instance + :param sha: bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def stream(self, sha): + """:return: OStream instance + :param sha: 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def size(self): + """:return: amount of objects in this database""" + raise NotImplementedError() + + def sha_iter(self): + """Return iterator yielding 20 byte shas for all objects in this data base""" + raise NotImplementedError() + + #} END query interface + + +class ObjectDBW: + + """Defines an interface to create objects in the database""" + + def __init__(self, *args, **kwargs): + self._ostream = None + + #{ Edit Interface + def set_ostream(self, stream): + """ + Adjusts the stream to which all data should be sent when storing new objects + + :param stream: if not None, the stream to use, if None the default stream + will be used. + :return: previously installed stream, or None if there was no override + :raise TypeError: if the stream doesn't have the supported functionality""" + cstream = self._ostream + self._ostream = stream + return cstream + + def ostream(self): + """ + Return the output stream + + :return: overridden output stream this instance will write to, or None + if it will write to the default stream""" + return self._ostream + + def store(self, istream): + """ + Create a new object in the database + :return: the input istream object with its sha set to its corresponding value + + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, + in which case the input stream is expected to be in object format ( header + contents ). + :raise IOError: if data could not be written""" + raise NotImplementedError("To be implemented in subclass") + + #} END edit interface + + +class FileDBBase: + + """Provides basic facilities to retrieve files of interest, including + caching facilities to help mapping hexsha's to objects""" + + def __init__(self, root_path): + """Initialize this instance to look for its files at the given root path + All subsequent operations will be relative to this path + :raise InvalidDBRoot: + **Note:** The base will not perform any accessablity checking as the base + might not yet be accessible, but become accessible before the first + access.""" + super().__init__() + self._root_path = root_path + + #{ Interface + def root_path(self): + """:return: path at which this db operates""" + return self._root_path + + def db_path(self, rela_path): + """ + :return: the given relative path relative to our database root, allowing + to pontentially access datafiles""" + return join(self._root_path, force_text(rela_path)) + #} END interface + + +class CachingDB: + + """A database which uses caches to speed-up access""" + + #{ Interface + def update_cache(self, force=False): + """ + Call this method if the underlying data changed to trigger an update + of the internal caching structures. + + :param force: if True, the update must be performed. Otherwise the implementation + may decide not to perform an update if it thinks nothing has changed. + :return: True if an update was performed as something change indeed""" + + # END interface + + +def _databases_recursive(database, output): + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + + +class CompoundDB(ObjectDBR, LazyMixin, CachingDB): + + """A database which delegates calls to sub-databases. + + Databases are stored in the lazy-loaded _dbs attribute. + Define _set_cache_ to update it with your databases""" + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + elif attr == '_db_cache': + self._db_cache = dict() + else: + super()._set_cache_(attr) + + def _db_query(self, sha): + """:return: database containing the given 20 byte sha + :raise BadObject:""" + # most databases use binary representations, prevent converting + # it every time a database is being queried + try: + return self._db_cache[sha] + except KeyError: + pass + # END first level cache + + for db in self._dbs: + if db.has_object(sha): + self._db_cache[sha] = db + return db + # END for each database + raise BadObject(sha) + + #{ ObjectDBR interface + + def has_object(self, sha): + try: + self._db_query(sha) + return True + except BadObject: + return False + # END handle exceptions + + def info(self, sha): + return self._db_query(sha).info(sha) + + def stream(self, sha): + return self._db_query(sha).stream(sha) + + def size(self): + """:return: total size of all contained databases""" + return reduce(lambda x, y: x + y, (db.size() for db in self._dbs), 0) + + def sha_iter(self): + return chain(*(db.sha_iter() for db in self._dbs)) + + #} END object DBR Interface + + #{ Interface + + def databases(self): + """:return: tuple of database instances we use for lookups""" + return tuple(self._dbs) + + def update_cache(self, force=False): + # something might have changed, clear everything + self._db_cache.clear() + stat = False + for db in self._dbs: + if isinstance(db, CachingDB): + stat |= db.update_cache(force) + # END if is caching db + # END for each database to update + return stat + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha (bytes or str) + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + partial_hexsha = force_text(partial_hexsha) + len_partial_hexsha = len(partial_hexsha) + if len_partial_hexsha % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if hasattr(db, 'partial_to_complete_sha_hex'): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + + #} END interface diff --git a/gitdb/gitdb/db/git.py b/gitdb/gitdb/db/git.py new file mode 100644 index 000000000..a1ed14280 --- /dev/null +++ b/gitdb/gitdb/db/git.py @@ -0,0 +1,85 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.db.base import ( + CompoundDB, + ObjectDBW, + FileDBBase +) + +from gitdb.db.loose import LooseObjectDB +from gitdb.db.pack import PackedDB +from gitdb.db.ref import ReferenceDB + +from gitdb.exc import InvalidDBRoot + +import os + +__all__ = ('GitDB', ) + + +class GitDB(FileDBBase, ObjectDBW, CompoundDB): + + """A git-style object database, which contains all objects in the 'objects' + subdirectory + + ``IMPORTANT``: The usage of this implementation is highly discouraged as it fails to release file-handles. + This can be a problem with long-running processes and/or big repositories. + """ + # Configuration + PackDBCls = PackedDB + LooseDBCls = LooseObjectDB + ReferenceDBCls = ReferenceDB + + # Directories + packs_dir = 'pack' + loose_dir = '' + alternates_dir = os.path.join('info', 'alternates') + + def __init__(self, root_path): + """Initialize ourselves on a git objects directory""" + super().__init__(root_path) + + def _set_cache_(self, attr): + if attr == '_dbs' or attr == '_loose_db': + self._dbs = list() + loose_db = None + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): + path = self.db_path(subpath) + if os.path.exists(path): + self._dbs.append(dbcls(path)) + if dbcls is self.LooseDBCls: + loose_db = self._dbs[-1] + # END remember loose db + # END check path exists + # END for each db type + + # should have at least one subdb + if not self._dbs: + raise InvalidDBRoot(self.root_path()) + # END handle error + + # we the first one should have the store method + assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" + + # finally set the value + self._loose_db = loose_db + else: + super()._set_cache_(attr) + # END handle attrs + + #{ ObjectDBW interface + + def store(self, istream): + return self._loose_db.store(istream) + + def ostream(self): + return self._loose_db.ostream() + + def set_ostream(self, ostream): + return self._loose_db.set_ostream(ostream) + + #} END objectdbw interface diff --git a/gitdb/gitdb/db/loose.py b/gitdb/gitdb/db/loose.py new file mode 100644 index 000000000..e6765cdeb --- /dev/null +++ b/gitdb/gitdb/db/loose.py @@ -0,0 +1,268 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from contextlib import suppress + +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + ObjectDBW +) + +from gitdb.exc import ( + BadObject, + AmbiguousObjectName +) + +from gitdb.stream import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + FDStream, + Sha1Writer +) + +from gitdb.base import ( + OStream, + OInfo +) + +from gitdb.util import ( + file_contents_ro_filepath, + ENOENT, + hex_to_bin, + bin_to_hex, + exists, + chmod, + isfile, + remove, + rename, + dirname, + basename, + join +) + +from gitdb.fun import ( + chunk_size, + loose_object_header_info, + write_object, + stream_copy +) + +from gitdb.utils.encoding import force_bytes + +import tempfile +import os +import sys +import time + + +__all__ = ('LooseObjectDB', ) + + +class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + # On windows we need to keep it writable, otherwise it cannot be removed + # either + new_objects_mode = int("444", 8) + if os.name == 'nt': + new_objects_mode = int("644", 8) + + def __init__(self, root_path): + super().__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + with suppress(KeyError): + return self._hexsha_to_file[hexsha] + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + def partial_to_complete_sha_hex(self, partial_hexsha): + """:return: 20 byte binary sha1 string which matches the given name uniquely + :param name: hexadecimal partial name (bytes or ascii string) + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for binsha in self.sha_iter(): + if bin_to_hex(binsha).startswith(force_bytes(partial_hexsha)): + # it can't ever find the same object twice + if candidate is not None: + raise AmbiguousObjectName(partial_hexsha) + candidate = binsha + # END for each object + if candidate is None: + raise BadObject(partial_hexsha) + return candidate + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(bin_to_hex(sha))) + try: + return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) + except OSError as e: + if e.errno != ENOENT: + # try again without noatime + try: + return file_contents_ro_filepath(db_path) + except OSError as new_e: + raise BadObject(sha) from new_e + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(sha) from e + # END handle error + # END exception handling + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super().set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + typ, size = loose_object_header_info(m) + return OInfo(sha, typ, size) + finally: + if hasattr(m, 'close'): + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion=True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(bin_to_hex(sha)) + return True + except BadObject: + return False + # END check existence + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + + if istream.binsha is None: + writer = FDCompressedSha1Writer(fd) + else: + writer = FDStream(fd) + # END handle direct stream copies + # END handle custom writer + + try: + try: + if istream.binsha is not None: + # copy as much as possible, the actual uncompressed item size might + # be smaller than the compressed version + stream_copy(istream.read, writer.write, sys.maxsize, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + except: + if tmp_path: + remove(tmp_path) + raise + # END assure tmpfile removal on error + + hexsha = None + if istream.binsha: + hexsha = istream.hexsha + else: + hexsha = writer.sha(as_hex=True) + # END handle sha + + if tmp_path: + obj_path = self.db_path(self.object_path(hexsha)) + obj_dir = dirname(obj_path) + os.makedirs(obj_dir, exist_ok=True) + # END handle destination directory + # rename onto existing doesn't work on NTFS + if isfile(obj_path): + remove(tmp_path) + else: + rename(tmp_path, obj_path) + # end rename only if needed + + # Ensure rename is actually done and file is stable + # Retry up to 14 times - quadratic wait & retry in ms. + # The total maximum wait time is 1000ms, which should be vastly enough for the + # OS to return and commit the file to disk. + for backoff_ms in [1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 181]: + with suppress(PermissionError): + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) + break + time.sleep(backoff_ms / 1000.0) + else: + raise PermissionError( + "Impossible to apply `chmod` to file {}".format(obj_path) + ) + + # END handle dry_run + + istream.binsha = hex_to_bin(hexsha) + return istream + + def sha_iter(self): + # find all files which look like an object, extract sha from there + for root, dirs, files in os.walk(self.root_path()): + root_base = basename(root) + if len(root_base) != 2: + continue + + for f in files: + if len(f) != 38: + continue + yield hex_to_bin(root_base + f) + # END for each file + # END for each walk iteration + + def size(self): + return len(tuple(self.sha_iter())) diff --git a/gitdb/gitdb/db/mem.py b/gitdb/gitdb/db/mem.py new file mode 100644 index 000000000..d4772fdb5 --- /dev/null +++ b/gitdb/gitdb/db/mem.py @@ -0,0 +1,110 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains the MemoryDatabase implementation""" +from gitdb.db.loose import LooseObjectDB +from gitdb.db.base import ( + ObjectDBR, + ObjectDBW +) + +from gitdb.base import ( + OStream, + IStream, +) + +from gitdb.exc import ( + BadObject, + UnsupportedOperation +) + +from gitdb.stream import ( + ZippedStoreShaWriter, + DecompressMemMapReader, +) + +from io import BytesIO + +__all__ = ("MemoryDB", ) + + +class MemoryDB(ObjectDBR, ObjectDBW): + + """A memory database stores everything to memory, providing fast IO and object + retrieval. It should be used to buffer results and obtain SHAs before writing + it to the actual physical storage, as it allows to query whether object already + exists in the target storage before introducing actual IO""" + + def __init__(self): + super().__init__() + self._db = LooseObjectDB("path/doesnt/matter") + + # maps 20 byte shas to their OStream objects + self._cache = dict() + + def set_ostream(self, stream): + raise UnsupportedOperation("MemoryDB's always stream into memory") + + def store(self, istream): + zstream = ZippedStoreShaWriter() + self._db.set_ostream(zstream) + + istream = self._db.store(istream) + zstream.close() # close to flush + zstream.seek(0) + + # don't provide a size, the stream is written in object format, hence the + # header needs decompression + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) + + return istream + + def has_object(self, sha): + return sha in self._cache + + def info(self, sha): + # we always return streams, which are infos as well + return self.stream(sha) + + def stream(self, sha): + try: + ostream = self._cache[sha] + # rewind stream for the next one to read + ostream.stream.seek(0) + return ostream + except KeyError as e: + raise BadObject(sha) from e + # END exception handling + + def size(self): + return len(self._cache) + + def sha_iter(self): + return self._cache.keys() + + #{ Interface + def stream_copy(self, sha_iter, odb): + """Copy the streams as identified by sha's yielded by sha_iter into the given odb + The streams will be copied directly + **Note:** the object will only be written if it did not exist in the target db + + :return: amount of streams actually copied into odb. If smaller than the amount + of input shas, one or more objects did already exist in odb""" + count = 0 + for sha in sha_iter: + if odb.has_object(sha): + continue + # END check object existence + + ostream = self.stream(sha) + # compressed data including header + sio = BytesIO(ostream.stream.data()) + istream = IStream(ostream.type, ostream.size, sio, sha) + + odb.store(istream) + count += 1 + # END for each sha + return count + #} END interface diff --git a/gitdb/gitdb/db/pack.py b/gitdb/gitdb/db/pack.py new file mode 100644 index 000000000..274ea5990 --- /dev/null +++ b/gitdb/gitdb/db/pack.py @@ -0,0 +1,206 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module containing a database to deal with packs""" +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + CachingDB +) + +from gitdb.util import LazyMixin + +from gitdb.exc import ( + BadObject, + UnsupportedOperation, + AmbiguousObjectName +) + +from gitdb.pack import PackEntity + +from functools import reduce + +import os +import glob + +__all__ = ('PackedDB', ) + +#{ Utilities + + +class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): + + """A database operating on a set of object packs""" + + # sort the priority list every N queries + # Higher values are better, performance tests don't show this has + # any effect, but it should have one + _sort_interval = 500 + + def __init__(self, root_path): + super().__init__(root_path) + # list of lists with three items: + # * hits - number of times the pack was hit with a request + # * entity - Pack entity instance + # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query + # self._entities = list() # lazy loaded list + self._hit_count = 0 # amount of hits + self._st_mtime = 0 # last modification data of our root path + + def _set_cache_(self, attr): + if attr == '_entities': + self._entities = list() + self.update_cache(force=True) + # END handle entities initialization + + def _sort_entities(self): + self._entities.sort(key=lambda l: l[0], reverse=True) + + def _pack_info(self, sha): + """:return: tuple(entity, index) for an item at the given sha + :param sha: 20 or 40 byte sha + :raise BadObject: + **Note:** This method is not thread-safe, but may be hit in multi-threaded + operation. The worst thing that can happen though is a counter that + was not incremented, or the list being in wrong order. So we safe + the time for locking here, lets see how that goes""" + # presort ? + if self._hit_count % self._sort_interval == 0: + self._sort_entities() + # END update sorting + + for item in self._entities: + index = item[2](sha) + if index is not None: + item[0] += 1 # one hit for you + self._hit_count += 1 # general hit count + return (item[1], index) + # END index found in pack + # END for each item + + # no hit, see whether we have to update packs + # NOTE: considering packs don't change very often, we safe this call + # and leave it to the super-caller to trigger that + raise BadObject(sha) + + #{ Object DB Read + + def has_object(self, sha): + try: + self._pack_info(sha) + return True + except BadObject: + return False + # END exception handling + + def info(self, sha): + entity, index = self._pack_info(sha) + return entity.info_at_index(index) + + def stream(self, sha): + entity, index = self._pack_info(sha) + return entity.stream_at_index(index) + + def sha_iter(self): + for entity in self.entities(): + index = entity.index() + sha_by_index = index.sha + for index in range(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x, y: x + y, sizes, 0) + + #} END object db read + + #{ object db write + + def store(self, istream): + """Storing individual objects is not feasible as a pack is designed to + hold multiple objects. Writing or rewriting packs for single objects is + inefficient""" + raise UnsupportedOperation() + + #} END object db write + + #{ Interface + + def update_cache(self, force=False): + """ + Update our cache with the actually existing packs on disk. Add new ones, + and remove deleted ones. We keep the unchanged ones + + :param force: If True, the cache will be updated even though the directory + does not appear to have changed according to its modification timestamp. + :return: True if the packs have been updated so there is new information, + False if there was no change to the pack database""" + stat = os.stat(self.root_path()) + if not force and stat.st_mtime <= self._st_mtime: + return False + # END abort early on no change + self._st_mtime = stat.st_mtime + + # packs are supposed to be prefixed with pack- by git-convention + # get all pack files, figure out what changed + pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) + our_pack_files = {item[1].pack().path() for item in self._entities} + + # new packs + for pack_file in (pack_files - our_pack_files): + # init the hit-counter/priority with the size, a good measure for hit- + # probability. Its implemented so that only 12 bytes will be read + entity = PackEntity(pack_file) + self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) + # END for each new packfile + + # removed packs + for pack_file in (our_pack_files - pack_files): + del_index = -1 + for i, item in enumerate(self._entities): + if item[1].pack().path() == pack_file: + del_index = i + break + # END found index + # END for each entity + assert del_index != -1 + del(self._entities[del_index]) + # END for each removed pack + + # reinitialize prioritiess + self._sort_entities() + return True + + def entities(self): + """:return: list of pack entities operated upon by this database""" + return [item[1] for item in self._entities] + + def partial_to_complete_sha(self, partial_binsha, canonical_length): + """:return: 20 byte sha as inferred by the given partial binary sha + :param partial_binsha: binary sha with less than 20 bytes + :param canonical_length: length of the corresponding canonical representation. + It is required as binary sha's cannot display whether the original hex sha + had an odd or even number of characters + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for item in self._entities: + item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) + if item_index is not None: + sha = item[1].index().sha(item_index) + if candidate and candidate != sha: + raise AmbiguousObjectName(partial_binsha) + candidate = sha + # END handle full sha could be found + # END for each entity + + if candidate: + return candidate + + # still not found ? + raise BadObject(partial_binsha) + + #} END interface diff --git a/gitdb/gitdb/db/ref.py b/gitdb/gitdb/db/ref.py new file mode 100644 index 000000000..5536db0f2 --- /dev/null +++ b/gitdb/gitdb/db/ref.py @@ -0,0 +1,81 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.db.base import ( + CompoundDB, +) + +__all__ = ('ReferenceDB', ) + + +class ReferenceDB(CompoundDB): + + """A database consisting of database referred to in a file""" + + # Configuration + # Specifies the object database to use for the paths found in the alternates + # file. If None, it defaults to the GitDB + ObjectDBCls = None + + def __init__(self, ref_file): + super().__init__() + self._ref_file = ref_file + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + self._update_dbs_from_ref_file() + else: + super()._set_cache_(attr) + # END handle attrs + + def _update_dbs_from_ref_file(self): + dbcls = self.ObjectDBCls + if dbcls is None: + # late import + from gitdb.db.git import GitDB + dbcls = GitDB + # END get db type + + # try to get as many as possible, don't fail if some are unavailable + ref_paths = list() + try: + with open(self._ref_file, 'r', encoding="utf-8") as f: + ref_paths = [l.strip() for l in f] + except OSError: + pass + # END handle alternates + + ref_paths_set = set(ref_paths) + cur_ref_paths_set = {db.root_path() for db in self._dbs} + + # remove existing + for path in (cur_ref_paths_set - ref_paths_set): + for i, db in enumerate(self._dbs[:]): + if db.root_path() == path: + del(self._dbs[i]) + continue + # END del matching db + # END for each path to remove + + # add new + # sort them to maintain order + added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) + for path in added_paths: + try: + db = dbcls(path) + # force an update to verify path + if isinstance(db, CompoundDB): + db.databases() + # END verification + self._dbs.append(db) + except Exception: + # ignore invalid paths or issues + pass + # END for each path to add + + def update_cache(self, force=False): + # re-read alternates and update databases + self._update_dbs_from_ref_file() + return super().update_cache(force) diff --git a/gitdb/gitdb/exc.py b/gitdb/gitdb/exc.py new file mode 100644 index 000000000..752dafdbb --- /dev/null +++ b/gitdb/gitdb/exc.py @@ -0,0 +1,57 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module with common exceptions""" +from gitdb.util import to_hex_sha + +__all__ = [ + 'AmbiguousObjectName', + 'BadName', + 'BadObject', + 'BadObjectType', + 'InvalidDBRoot', + 'ODBError', + 'ParseError', + 'UnsupportedOperation', + 'to_hex_sha', +] + +class ODBError(Exception): + """All errors thrown by the object database""" + + +class InvalidDBRoot(ODBError): + """Thrown if an object database cannot be initialized at the given path""" + + +class BadObject(ODBError): + """The object with the given SHA does not exist. Instantiate with the + failed sha""" + + def __str__(self): + return "BadObject: %s" % to_hex_sha(self.args[0]) + + +class BadName(ODBError): + """A name provided to rev_parse wasn't understood""" + + def __str__(self): + return "Ref '%s' did not resolve to an object" % self.args[0] + + +class ParseError(ODBError): + """Thrown if the parsing of a file failed due to an invalid format""" + + +class AmbiguousObjectName(ODBError): + """Thrown if a possibly shortened name does not uniquely represent a single object + in the database""" + + +class BadObjectType(ODBError): + """The object had an unsupported type""" + + +class UnsupportedOperation(ODBError): + """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/gitdb/fun.py b/gitdb/gitdb/fun.py new file mode 100644 index 000000000..a272e5caa --- /dev/null +++ b/gitdb/gitdb/fun.py @@ -0,0 +1,704 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains basic c-functions which usually contain performance critical code +Keeping this code separate from the beginning makes it easier to out-source +it into c later, if required""" + +import zlib +from gitdb.util import byte_ord +decompressobj = zlib.decompressobj + +import mmap +from itertools import islice +from functools import reduce + +from gitdb.const import NULL_BYTE, BYTE_SPACE +from gitdb.utils.encoding import force_text +from gitdb.typ import ( + str_blob_type, + str_commit_type, + str_tree_type, + str_tag_type, +) + +from io import StringIO + +# INVARIANTS +OFS_DELTA = 6 +REF_DELTA = 7 +delta_types = (OFS_DELTA, REF_DELTA) + +type_id_to_type_map = { + 0: b'', # EXT 1 + 1: str_commit_type, + 2: str_tree_type, + 3: str_blob_type, + 4: str_tag_type, + 5: b'', # EXT 2 + OFS_DELTA: "OFS_DELTA", # OFFSET DELTA + REF_DELTA: "REF_DELTA" # REFERENCE DELTA +} + +type_to_type_id_map = { + str_commit_type: 1, + str_tree_type: 2, + str_blob_type: 3, + str_tag_type: 4, + "OFS_DELTA": OFS_DELTA, + "REF_DELTA": REF_DELTA, +} + +# used when dealing with larger streams +chunk_size = 1000 * mmap.PAGESIZE + +__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') + + +#{ Structures + +def _set_delta_rbound(d, size): + """Truncate the given delta to the given size + :param size: size relative to our target offset, may not be 0, must be smaller or equal + to our size + :return: d""" + d.ts = size + + # NOTE: data is truncated automatically when applying the delta + # MUST NOT DO THIS HERE + return d + + +def _move_delta_lbound(d, bytes): + """Move the delta by the given amount of bytes, reducing its size so that its + right bound stays static + :param bytes: amount of bytes to move, must be smaller than delta size + :return: d""" + if bytes == 0: + return + + d.to += bytes + d.so += bytes + d.ts -= bytes + if d.data is not None: + d.data = d.data[bytes:] + # END handle data + + return d + + +def delta_duplicate(src): + return DeltaChunk(src.to, src.ts, src.so, src.data) + + +def delta_chunk_apply(dc, bbuf, write): + """Apply own data to the target buffer + :param bbuf: buffer providing source bytes for copy operations + :param write: write method to call with data to write""" + if dc.data is None: + # COPY DATA FROM SOURCE + write(bbuf[dc.so:dc.so + dc.ts]) + else: + # APPEND DATA + # what's faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it + if dc.ts < len(dc.data): + write(dc.data[:dc.ts]) + else: + write(dc.data) + # END handle truncation + # END handle chunk mode + + +class DeltaChunk: + + """Represents a piece of a delta, it can either add new data, or copy existing + one from a source buffer""" + __slots__ = ( + 'to', # start offset in the target buffer in bytes + 'ts', # size of this chunk in the target buffer in bytes + 'so', # start offset in the source buffer in bytes or None + 'data', # chunk of bytes to be added to the target buffer, + # DeltaChunkList to use as base, or None + ) + + def __init__(self, to, ts, so, data): + self.to = to + self.ts = ts + self.so = so + self.data = data + + def __repr__(self): + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + + #{ Interface + + def rbound(self): + return self.to + self.ts + + def has_data(self): + """:return: True if the instance has data to add to the target stream""" + return self.data is not None + + #} END interface + + +def _closest_index(dcl, absofs): + """:return: index at which the given absofs should be inserted. The index points + to the DeltaChunk with a target buffer absofs that equals or is greater than + absofs. + **Note:** global method for performance only, it belongs to DeltaChunkList""" + lo = 0 + hi = len(dcl) + while lo < hi: + mid = (lo + hi) / 2 + dc = dcl[mid] + if dc.to > absofs: + hi = mid + elif dc.rbound() > absofs or dc.to == absofs: + return mid + else: + lo = mid + 1 + # END handle bound + # END for each delta absofs + return len(dcl) - 1 + + +def delta_list_apply(dcl, bbuf, write): + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param write: function taking a string of bytes to write to the output""" + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc + + +def delta_list_slice(dcl, absofs, size, ndcl): + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: None""" + cdi = _closest_index(dcl, absofs) # delta start index + cd = dcl[cdi] + slen = len(dcl) + lappend = ndcl.append + + if cd.to != absofs: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + _move_delta_lbound(tcd, absofs - cd.to) + tcd.ts = min(tcd.ts, size) + lappend(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: + # are we larger than the current block + cd = dcl[cdi] + if cd.ts <= size: + lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) + size -= cd.ts + else: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + tcd.ts = size + lappend(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 + # END for each chunk + + +class DeltaChunkList(list): + + """List with special functionality to deal with DeltaChunks. + There are two types of lists we represent. The one was created bottom-up, working + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable + after all processing with is_reversed.""" + + __slots__ = tuple() + + def rbound(self): + """:return: rightmost extend in bytes, absolute""" + if len(self) == 0: + return 0 + return self[-1].rbound() + + def lbound(self): + """:return: leftmost byte at which this chunklist starts""" + if len(self) == 0: + return 0 + return self[0].to + + def size(self): + """:return: size of bytes as measured by our delta chunks""" + return self.rbound() - self.lbound() + + def apply(self, bbuf, write): + """Only used by public clients, internally we only use the global routines + for performance""" + return delta_list_apply(self, bbuf, write) + + def compress(self): + """Alter the list to reduce the amount of nodes. Currently we concatenate + add-chunks + :return: self""" + slen = len(self) + if slen < 2: + return self + i = 0 + + first_data_index = None + while i < slen: + dc = self[i] + i += 1 + if dc.data is None: + if first_data_index is not None and i - 2 - first_data_index > 1: + # if first_data_index is not None: + nd = StringIO() # new data + so = self[first_data_index].to # start offset in target buffer + for x in range(first_data_index, i - 1): + xdc = self[x] + nd.write(xdc.data[:xdc.ts]) + # END collect data + + del(self[first_data_index:i - 1]) + buf = nd.getvalue() + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + + slen = len(self) + i = first_data_index + 1 + + # END concatenate data + first_data_index = None + continue + # END skip non-data chunks + + if first_data_index is None: + first_data_index = i - 1 + # END iterate list + + # if slen_orig != len(self): + # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) + return self + + def check_integrity(self, target_size=-1): + """Verify the list has non-overlapping chunks only, and the total size matches + target_size + :param target_size: if not -1, the total size of the chain must be target_size + :raise AssertionError: if the size doesn't match""" + if target_size > -1: + assert self[-1].rbound() == target_size + assert reduce(lambda x, y: x + y, (d.ts for d in self), 0) == target_size + # END target size verification + + if len(self) < 2: + return + + # check data + for dc in self: + assert dc.ts > 0 + if dc.has_data(): + assert len(dc.data) >= dc.ts + # END for each dc + + left = islice(self, 0, len(self) - 1) + right = iter(self) + right.next() + # this is very pythonic - we might have just use index based access here, + # but this could actually be faster + for lft, rgt in zip(left, right): + assert lft.rbound() == rgt.to + assert lft.to + lft.ts == rgt.to + # END for each pair + + +class TopdownDeltaChunkList(DeltaChunkList): + + """Represents a list which is generated by feeding its ancestor streams one by + one""" + __slots__ = tuple() + + def connect_with_next_base(self, bdcl): + """Connect this chain with the next level of our base delta chunklist. + The goal in this game is to mark as many of our chunks rigid, hence they + cannot be changed by any of the upcoming bases anymore. Once all our + chunks are marked like that, we can stop all processing + :param bdcl: data chunk list being one of our bases. They must be fed in + consecutively and in order, towards the earliest ancestor delta + :return: True if processing was done. Use it to abort processing of + remaining streams if False is returned""" + nfc = 0 # number of frozen chunks + dci = 0 # delta chunk index + slen = len(self) # len of self + ccl = list() # temporary list + while dci < slen: + dc = self[dci] + dci += 1 + + # all add-chunks which are already topmost don't need additional processing + if dc.data is not None: + nfc += 1 + continue + # END skip add chunks + + # copy chunks + # integrate the portion of the base list into ourselves. Lists + # dont support efficient insertion ( just one at a time ), but for now + # we live with it. Internally, its all just a 32/64bit pointer, and + # the portions of moved memory should be smallish. Maybe we just rebuild + # ourselves in order to reduce the amount of insertions ... + del(ccl[:]) + delta_list_slice(bdcl, dc.so, dc.ts, ccl) + + # move the target bounds into place to match with our chunk + ofs = dc.to - dc.so + for cdc in ccl: + cdc.to += ofs + # END update target bounds + + if len(ccl) == 1: + self[dci - 1] = ccl[0] + else: + # maybe try to compute the expenses here, and pick the right algorithm + # It would normally be faster than copying everything physically though + # TODO: Use a deque here, and decide by the index whether to extend + # or extend left ! + post_dci = self[dci:] + del(self[dci - 1:]) # include deletion of dc + self.extend(ccl) + self.extend(post_dci) + + slen = len(self) + dci += len(ccl) - 1 # deleted dc, added rest + + # END handle chunk replacement + # END for each chunk + + if nfc == slen: + return False + # END handle completeness + return True + + +#} END structures + +#{ Routines + +def is_loose_object(m): + """ + :return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" + b0, b1 = map(ord, m[:2]) + word = (b0 << 8) + b1 + return b0 == 0x78 and (word % 31) == 0 + + +def loose_object_header_info(m): + """ + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + object as well as its uncompressed size in bytes. + :param m: memory map from which to read the compressed object data""" + decompress_size = 8192 # is used in cgit as well + hdr = decompressobj().decompress(m, decompress_size) + type_name, size = hdr[:hdr.find(NULL_BYTE)].split(BYTE_SPACE) + + return type_name, int(size) + + +def pack_object_header_info(data): + """ + :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream + :param m: random-access memory, like a string or memory map""" + c = byte_ord(data[0]) # first byte + i = 1 # next char to read + type_id = (c >> 4) & 7 # numeric type + size = c & 15 # starting size + s = 4 # starting bit-shift size + while c & 0x80: + c = byte_ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + # end performance at expense of maintenance ... + return (type_id, size, i) + + +def create_pack_object_header(obj_type, obj_size): + """ + :return: string defining the pack header comprised of the object type + and its incompressed size in bytes + + :param obj_type: pack type_id of the object + :param obj_size: uncompressed size in bytes of the following object stream""" + c = 0 # 1 byte + hdr = bytearray() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr.append(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + # END until size is consumed + hdr.append(c) + # end handle interpreter + return hdr + + +def msb_size(data, offset=0): + """ + :return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" + size = 0 + i = 0 + l = len(data) + hit_msb = False + while i < l: + c = data[i + offset] + size |= (c & 0x7f) << i * 7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + # end performance ... + if not hit_msb: + raise AssertionError("Could not find terminating MSB byte in data stream") + return i + offset, size + + +def loose_object_header(type, size): + """ + :return: bytes representing the loose object header, which is immediately + followed by the content stream of size 'size'""" + return ('%s %i\0' % (force_text(type), size)).encode('ascii') + + +def write_object(type, size, read, write, chunk_size=chunk_size): + """ + Write the object as identified by type, size and source_stream into the + target_stream + + :param type: type string of the object + :param size: amount of bytes to write from source_stream + :param read: read method of a stream providing the content data + :param write: write method of the output stream + :param close_target_stream: if True, the target stream will be closed when + the routine exits, even if an error is thrown + :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" + tbw = 0 # total num bytes written + + # WRITE HEADER: type SP size NULL + tbw += write(loose_object_header(type, size)) + tbw += stream_copy(read, write, size, chunk_size) + + return tbw + + +def stream_copy(read, write, size, chunk_size): + """ + Copy a stream up to size bytes using the provided read and write methods, + in chunks of chunk_size + + **Note:** its much like stream_copy utility, but operates just using methods""" + dbw = 0 # num data bytes written + + # WRITE ALL DATA UP TO SIZE + while True: + cs = min(chunk_size, size - dbw) + # NOTE: not all write methods return the amount of written bytes, like + # mmap.write. Its bad, but we just deal with it ... perhaps its not + # even less efficient + # data_len = write(read(cs)) + # dbw += data_len + data = read(cs) + data_len = len(data) + dbw += data_len + write(data) + if data_len < cs or dbw == size: + break + # END check for stream end + # END duplicate data + return dbw + + +def connect_deltas(dstreams): + """ + Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + + :param dstreams: iterable of delta stream objects, the delta to be applied last + comes first, then all its ancestors in order + :return: DeltaChunkList, containing all operations to apply""" + tdcl = None # topmost dcl + + dcl = tdcl = TopdownDeltaChunkList() + for dsi, ds in enumerate(dstreams): + # print "Stream", dsi + db = ds.read() + delta_buf_size = ds.size + + # read header + i, base_size = msb_size(db) + i, target_size = msb_size(db, i) + + # interpret opcodes + tbw = 0 # amount of target bytes written + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > base_size): + break + + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) + tbw += cp_size + elif c: + # NOTE: in C, the data chunks should probably be concatenated here. + # In python, we do it as a post-process + dcl.append(DeltaChunk(tbw, c, 0, db[i:i + c])) + i += c + tbw += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + dcl.compress() + + # merge the lists ! + if dsi > 0: + if not tdcl.connect_with_next_base(dcl): + break + # END handle merge + + # prepare next base + dcl = DeltaChunkList() + # END for each delta stream + + return tdcl + + +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): + """ + Apply data from a delta buffer using a source buffer to the target file + + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size for the delta buffer in bytes + :param delta_buf: random access delta data + :param write: write method taking a chunk of bytes + + **Note:** transcribed to python from the similar routine in patch-delta.c""" + i = 0 + db = delta_buf + while i < delta_buf_size: + c = db[i] + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = db[i] + i += 1 + if (c & 0x02): + cp_off |= (db[i] << 8) + i += 1 + if (c & 0x04): + cp_off |= (db[i] << 16) + i += 1 + if (c & 0x08): + cp_off |= (db[i] << 24) + i += 1 + if (c & 0x10): + cp_size = db[i] + i += 1 + if (c & 0x20): + cp_size |= (db[i] << 8) + i += 1 + if (c & 0x40): + cp_size |= (db[i] << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(src_buf[cp_off:cp_off + cp_size]) + elif c: + write(db[i:i + c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + # yes, lets use the exact same error message that git uses :) + assert i == delta_buf_size, "delta replay has gone wild" + + +def is_equal_canonical_sha(canonical_length, match, sha1): + """ + :return: True if the given lhs and rhs 20 byte binary shas + The comparison will take the canonical_length of the match sha into account, + hence the comparison will only use the last 4 bytes for uneven canonical representations + :param match: less than 20 byte sha + :param sha1: 20 byte sha""" + binary_length = canonical_length // 2 + if match[:binary_length] != sha1[:binary_length]: + return False + + if canonical_length - binary_length and \ + (byte_ord(match[-1]) ^ byte_ord(sha1[len(match) - 1])) & 0xf0: + return False + # END handle uneven canonnical length + return True + +#} END routines + + +try: + from gitdb_speedups._perf import connect_deltas +except ImportError: + pass diff --git a/gitdb/gitdb/pack.py b/gitdb/gitdb/pack.py new file mode 100644 index 000000000..2e0707947 --- /dev/null +++ b/gitdb/gitdb/pack.py @@ -0,0 +1,1028 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains PackIndexFile and PackFile implementations""" +import zlib + +from gitdb.exc import ( + BadObject, + AmbiguousObjectName, + UnsupportedOperation, + ParseError +) + +from gitdb.util import ( + mman, + LazyMixin, + unpack_from, + bin_to_hex, + byte_ord, +) + +from gitdb.fun import ( + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size +) + +try: + from gitdb_speedups._perf import PackIndexFile_sha_to_index +except ImportError: + pass +# END try c module + +from gitdb.base import ( # Amazing ! + OInfo, + OStream, + OPackInfo, + OPackStream, + ODeltaStream, + ODeltaPackInfo, + ODeltaPackStream, +) + +from gitdb.stream import ( + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer +) + +from struct import pack +from binascii import crc32 + +from gitdb.const import NULL_BYTE + +import tempfile +import array +import os +import sys + +__all__ = ('PackIndexFile', 'PackFile', 'PackEntity') + + +#{ Utilities + +def pack_object_at(cursor, offset, as_stream): + """ + :return: Tuple(abs_data_offset, PackInfo|PackStream) + an object of the correct type according to the type_id of the object. + If as_stream is True, the object will contain a stream, allowing the + data to be read decompressed. + :param data: random accessible data containing all required information + :parma offset: offset in to the data at which the object information is located + :param as_stream: if True, a stream object will be returned that can read + the data, otherwise you receive an info object only""" + data = cursor.use_region(offset).buffer() + type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) + total_rela_offset = None # set later, actual offset until data stream begins + delta_info = None + + # OFFSET DELTA + if type_id == OFS_DELTA: + i = data_rela_offset + c = byte_ord(data[i]) + i += 1 + delta_offset = c & 0x7f + while c & 0x80: + c = byte_ord(data[i]) + i += 1 + delta_offset += 1 + delta_offset = (delta_offset << 7) + (c & 0x7f) + # END character loop + delta_info = delta_offset + total_rela_offset = i + # REF DELTA + elif type_id == REF_DELTA: + total_rela_offset = data_rela_offset + 20 + delta_info = data[data_rela_offset:total_rela_offset] + # BASE OBJECT + else: + # assume its a base object + total_rela_offset = data_rela_offset + # END handle type id + abs_data_offset = offset + total_rela_offset + if as_stream: + stream = DecompressMemMapReader(data[total_rela_offset:], False, uncomp_size) + if delta_info is None: + return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) + else: + return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) + else: + if delta_info is None: + return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) + else: + return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) + # END handle info + # END handle stream + + +def write_stream_to_pack(read, write, zstream, base_crc=None): + """Copy a stream as read from read function, zip it, and write the result. + Count the number of written bytes and return it + :param base_crc: if not None, the crc will be the base for all compressed data + we consecutively write and generate a crc32 from. If None, no crc will be generated + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc + was false""" + br = 0 # bytes read + bw = 0 # bytes written + want_crc = base_crc is not None + crc = 0 + if want_crc: + crc = base_crc + # END initialize crc + + while True: + chunk = read(chunk_size) + br += len(chunk) + compressed = zstream.compress(chunk) + bw += len(compressed) + write(compressed) # cannot assume return value + + if want_crc: + crc = crc32(compressed, crc) + # END handle crc + + if len(chunk) != chunk_size: + break + # END copy loop + + compressed = zstream.flush() + bw += len(compressed) + write(compressed) + if want_crc: + crc = crc32(compressed, crc) + # END handle crc + + return (br, bw, crc) + + +#} END utilities + + +class IndexWriter: + + """Utility to cache index information, allowing to write all information later + in one go to the given stream + **Note:** currently only writes v2 indices""" + __slots__ = '_objs' + + def __init__(self): + self._objs = list() + + def append(self, binsha, crc, offset): + """Append one piece of object information""" + self._objs.append((binsha, crc, offset)) + + def write(self, pack_sha, write): + """Write the index file using the given write method + :param pack_sha: binary sha over the whole pack that we index + :return: sha1 binary sha over all index file contents""" + # sort for sha1 hash + self._objs.sort(key=lambda o: o[0]) + + sha_writer = FlexibleSha1Writer(write) + sha_write = sha_writer.write + sha_write(PackIndexFile.index_v2_signature) + sha_write(pack(">L", PackIndexFile.index_version_default)) + + # fanout + tmplist = list((0,) * 256) # fanout or list with 64 bit offsets + for t in self._objs: + tmplist[byte_ord(t[0][0])] += 1 + # END prepare fanout + for i in range(255): + v = tmplist[i] + sha_write(pack('>L', v)) + tmplist[i + 1] += v + # END write each fanout entry + sha_write(pack('>L', tmplist[255])) + + # sha1 ordered + # save calls, that is push them into c + sha_write(b''.join(t[0] for t in self._objs)) + + # crc32 + for t in self._objs: + sha_write(pack('>L', t[1] & 0xffffffff)) + # END for each crc + + tmplist = list() + # offset 32 + for t in self._objs: + ofs = t[2] + if ofs > 0x7fffffff: + tmplist.append(ofs) + ofs = 0x80000000 + len(tmplist) - 1 + # END handle 64 bit offsets + sha_write(pack('>L', ofs & 0xffffffff)) + # END for each offset + + # offset 64 + for ofs in tmplist: + sha_write(pack(">Q", ofs)) + # END for each offset + + # trailer + assert(len(pack_sha) == 20) + sha_write(pack_sha) + sha = sha_writer.sha(as_hex=False) + write(sha) + return sha + + +class PackIndexFile(LazyMixin): + + """A pack index provides offsets into the corresponding pack, allowing to find + locations for offsets faster.""" + + # Dont use slots as we dynamically bind functions for each version, need a dict for this + # The slots you see here are just to keep track of our instance variables + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') + + # used in v2 indices + _sha_list_offset = 8 + 1024 + index_v2_signature = b'\xfftOc' + index_version_default = 2 + + def __init__(self, indexpath): + super().__init__() + self._indexpath = indexpath + + def close(self): + mman.force_map_handle_removal_win(self._indexpath) + self._cursor = None + + def _set_cache_(self, attr): + if attr == "_packfile_checksum": + self._packfile_checksum = self._cursor.map()[-40:-20] + elif attr == "_cursor": + # Note: We don't lock the file when reading as we cannot be sure + # that we can actually write to the location - it could be a read-only + # alternate for instance + self._cursor = mman.make_cursor(self._indexpath).use_region() + # We will assume that the index will always fully fit into memory ! + if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % ( + self._indexpath, self._cursor.file_size(), mman.window_size())) + # END assert window size + else: + # now its time to initialize everything - if we are here, someone wants + # to access the fanout table or related properties + + # CHECK VERSION + mmap = self._cursor.map() + self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 + if self._version == 2: + version_id = unpack_from(">L", mmap, 4)[0] + assert version_id == self._version, "Unsupported index version: %i" % version_id + # END assert version + + # SETUP FUNCTIONS + # setup our functions according to the actual version + for fname in ('entry', 'offset', 'sha', 'crc'): + setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) + # END for each function to initialize + + # INITIALIZE DATA + # byte offset is 8 if version is 2, 0 otherwise + self._initialize() + # END handle attributes + + #{ Access V1 + + def _entry_v1(self, i): + """:return: tuple(offset, binsha, 0)""" + return unpack_from(">L20s", self._cursor.map(), 1024 + i * 24) + (0, ) + + def _offset_v1(self, i): + """see ``_offset_v2``""" + return unpack_from(">L", self._cursor.map(), 1024 + i * 24)[0] + + def _sha_v1(self, i): + """see ``_sha_v2``""" + base = 1024 + (i * 24) + 4 + return self._cursor.map()[base:base + 20] + + def _crc_v1(self, i): + """unsupported""" + return 0 + + #} END access V1 + + #{ Access V2 + def _entry_v2(self, i): + """:return: tuple(offset, binsha, crc)""" + return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) + + def _offset_v2(self, i): + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + be returned if the pack is larger than 4 GiB, or 2^32""" + offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] + + # if the high-bit is set, this indicates that we have to lookup the offset + # in the 64 bit region of the file. The current offset ( lower 31 bits ) + # are the index into it + if offset & 0x80000000: + offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] + # END handle 64 bit offset + + return offset + + def _sha_v2(self, i): + """:return: sha at the given index of this file index instance""" + base = self._sha_list_offset + i * 20 + return self._cursor.map()[base:base + 20] + + def _crc_v2(self, i): + """:return: 4 bytes crc for the object at index i""" + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + + #} END access V2 + + #{ Initialization + + def _initialize(self): + """initialize base data""" + self._fanout_table = self._read_fanout((self._version == 2) * 8) + + if self._version == 2: + self._crc_list_offset = self._sha_list_offset + self.size() * 20 + self._pack_offset = self._crc_list_offset + self.size() * 4 + self._pack_64_offset = self._pack_offset + self.size() * 4 + # END setup base + + def _read_fanout(self, byte_offset): + """Generate a fanout table from our data""" + d = self._cursor.map() + out = list() + append = out.append + for i in range(256): + append(unpack_from('>L', d, byte_offset + i * 4)[0]) + # END for each entry + return out + + #} END initialization + + #{ Properties + def version(self): + return self._version + + def size(self): + """:return: amount of objects referred to by this index""" + return self._fanout_table[255] + + def path(self): + """:return: path to the packindexfile""" + return self._indexpath + + def packfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of the pack file""" + return self._cursor.map()[-40:-20] + + def indexfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of this index file""" + return self._cursor.map()[-20:] + + def offsets(self): + """:return: sequence of all offsets in the order in which they were written + + **Note:** return value can be random accessed, but may be immmutable""" + if self._version == 2: + # read stream to array, convert to tuple + a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears + a.frombytes(self._cursor.map()[self._pack_offset:self._pack_64_offset]) + + # networkbyteorder to something array likes more + if sys.byteorder == 'little': + a.byteswap() + return a + else: + return tuple(self.offset(index) for index in range(self.size())) + # END handle version + + def sha_to_index(self, sha): + """ + :return: index usable with the ``offset`` or ``entry`` method, or None + if the sha was not found in this pack index + :param sha: 20 byte sha to lookup""" + first_byte = byte_ord(sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte - 1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # bisect until we have the sha + while lo < hi: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if sha < mid_sha: + hi = mid + elif sha == mid_sha: + return mid + else: + lo = mid + 1 + # END handle midpoint + # END bisect + return None + + def partial_sha_to_index(self, partial_bin_sha, canonical_length): + """ + :return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_bin_sha: an at least two bytes of a partial binary sha as bytes + :param canonical_length: length of the original hexadecimal representation of the + given partial binary sha + :raise AmbiguousObjectName:""" + if len(partial_bin_sha) < 2: + raise ValueError("Require at least 2 bytes of partial sha") + + assert isinstance(partial_bin_sha, bytes), "partial_bin_sha must be bytes" + first_byte = byte_ord(partial_bin_sha[0]) + + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte - 1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # fill the partial to full 20 bytes + filled_sha = partial_bin_sha + NULL_BYTE * (20 - len(partial_bin_sha)) + + # find lowest + while lo < hi: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if filled_sha < mid_sha: + hi = mid + elif filled_sha == mid_sha: + # perfect match + lo = mid + break + else: + lo = mid + 1 + # END handle midpoint + # END bisect + + if lo < self.size(): + cur_sha = get_sha(lo) + if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): + next_sha = None + if lo + 1 < self.size(): + next_sha = get_sha(lo + 1) + if next_sha and next_sha == cur_sha: + raise AmbiguousObjectName(partial_bin_sha) + return lo + # END if we have a match + # END if we found something + return None + + if 'PackIndexFile_sha_to_index' in globals(): + # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # accesses + def sha_to_index(self, sha): + return PackIndexFile_sha_to_index(self, sha) + # END redefine heavy-hitter with c version + + #} END properties + + +class PackFile(LazyMixin): + + """A pack is a file written according to the Version 2 for git packs + + As we currently use memory maps, it could be assumed that the maximum size of + packs therefore is 32 bit on 32 bit systems. On 64 bit systems, this should be + fine though. + + **Note:** at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" + + __slots__ = ('_packpath', '_cursor', '_size', '_version') + pack_signature = 0x5041434b # 'PACK' + pack_version_default = 2 + + # offset into our data at which the first object starts + first_object_offset = 3 * 4 # header bytes + footer_size = 20 # final sha + + def __init__(self, packpath): + self._packpath = packpath + + def close(self): + mman.force_map_handle_removal_win(self._packpath) + self._cursor = None + + def _set_cache_(self, attr): + # we fill the whole cache, whichever attribute gets queried first + self._cursor = mman.make_cursor(self._packpath).use_region() + + # read the header information + type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + if type_id != self.pack_signature: + raise ParseError("Invalid pack signature: %i" % type_id) + + def _iter_objects(self, start_offset, as_stream=True): + """Handle the actual iteration of objects within this pack""" + c = self._cursor + content_size = c.file_size() - self.footer_size + cur_offset = start_offset or self.first_object_offset + + null = NullStream() + while cur_offset < content_size: + data_offset, ostream = pack_object_at(c, cur_offset, True) + # scrub the stream to the end - this decompresses the object, but yields + # the amount of compressed bytes we need to get to the next offset + + stream_copy(ostream.read, null.write, ostream.size, chunk_size) + assert ostream.stream._br == ostream.size + cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() + + # if a stream is requested, reset it beforehand + # Otherwise return the Stream object directly, its derived from the + # info object + if as_stream: + ostream.stream.seek(0) + yield ostream + # END until we have read everything + + #{ Pack Information + + def size(self): + """:return: The amount of objects stored in this pack""" + return self._size + + def version(self): + """:return: the version of this pack""" + return self._version + + def data(self): + """ + :return: read-only data of this pack. It provides random access and usually + is a memory map. + :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" + # can use map as we are starting at offset 0. Otherwise we would have to use buffer() + return self._cursor.use_region().map() + + def checksum(self): + """:return: 20 byte sha1 hash on all object sha's contained in this file""" + return self._cursor.use_region(self._cursor.file_size() - 20).buffer()[:] + + def path(self): + """:return: path to the packfile""" + return self._packpath + #} END pack information + + #{ Pack Specific + + def collect_streams(self, offset): + """ + :return: list of pack streams which are required to build the object + at the given offset. The first entry of the list is the object at offset, + the last one is either a full object, or a REF_Delta stream. The latter + type needs its reference object to be locked up in an ODB to form a valid + delta chain. + If the object at offset is no delta, the size of the list is 1. + :param offset: specifies the first byte of the object within this pack""" + out = list() + c = self._cursor + while True: + ostream = pack_object_at(c, offset, True)[1] + out.append(ostream) + if ostream.type_id == OFS_DELTA: + offset = ostream.pack_offset - ostream.delta_info + else: + # the only thing we can lookup are OFFSET deltas. Everything + # else is either an object, or a ref delta, in the latter + # case someone else has to find it + break + # END handle type + # END while chaining streams + return out + + #} END pack specific + + #{ Read-Database like Interface + + def info(self, offset): + """Retrieve information about the object at the given file-absolute offset + + :param offset: byte offset + :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] + + def stream(self, offset): + """Retrieve an object at the given file-relative offset as stream along with its information + + :param offset: byte offset + :return: OPackStream instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] + + def stream_iter(self, start_offset=0): + """ + :return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. + :param start_offset: offset to the first object to iterate. If 0, iteration + starts at the very first object in the pack. + + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" + return self._iter_objects(start_offset, as_stream=True) + + #} END Read-Database like Interface + + +class PackEntity(LazyMixin): + + """Combines the PackIndexFile and the PackFile into one, allowing the + actual objects to be resolved and iterated""" + + __slots__ = ('_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) + + IndexFileCls = PackIndexFile + PackFileCls = PackFile + + def __init__(self, pack_or_index_path): + """Initialize ourselves with the path to the respective pack or index file""" + basename, ext = os.path.splitext(pack_or_index_path) + self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance + self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + def close(self): + self._index.close() + self._pack.close() + + def _set_cache_(self, attr): + # currently this can only be _offset_map + # TODO: make this a simple sorted offset array which can be bisected + # to find the respective entry, from which we can take a +1 easily + # This might be slower, but should also be much lighter in memory ! + offsets_sorted = sorted(self._index.offsets()) + last_offset = len(self._pack.data()) - self._pack.footer_size + assert offsets_sorted, "Cannot handle empty indices" + + offset_map = None + if len(offsets_sorted) == 1: + offset_map = {offsets_sorted[0]: last_offset} + else: + iter_offsets = iter(offsets_sorted) + iter_offsets_plus_one = iter(offsets_sorted) + next(iter_offsets_plus_one) + consecutive = zip(iter_offsets, iter_offsets_plus_one) + + offset_map = dict(consecutive) + + # the last offset is not yet set + offset_map[offsets_sorted[-1]] = last_offset + # END handle offset amount + self._offset_map = offset_map + + def _sha_to_index(self, sha): + """:return: index for the given sha, or raise""" + index = self._index.sha_to_index(sha) + if index is None: + raise BadObject(sha) + return index + + def _iter_objects(self, as_stream): + """Iterate over all objects in our index and yield their OInfo or OStream instences""" + _sha = self._index.sha + _object = self._object + for index in range(self._index.size()): + yield _object(_sha(index), as_stream, index) + # END for each index + + def _object(self, sha, as_stream, index=-1): + """:return: OInfo or OStream object providing information about the given sha + :param index: if not -1, its assumed to be the sha's index in the IndexFile""" + # its a little bit redundant here, but it needs to be efficient + if index < 0: + index = self._sha_to_index(sha) + if sha is None: + sha = self._index.sha(index) + # END assure sha is present ( in output ) + offset = self._index.offset(index) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) + if as_stream: + if type_id not in delta_types: + packstream = self._pack.stream(offset) + return OStream(sha, packstream.type, packstream.size, packstream.stream) + # END handle non-deltas + + # produce a delta stream containing all info + # To prevent it from applying the deltas when querying the size, + # we extract it from the delta stream ourselves + streams = self.collect_streams_at_offset(offset) + dstream = DeltaApplyReader.new(streams) + + return ODeltaStream(sha, dstream.type, None, dstream) + else: + if type_id not in delta_types: + return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) + # END handle non-deltas + + # deltas are a little tougher - unpack the first bytes to obtain + # the actual target size, as opposed to the size of the delta data + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + # collect the streams to obtain the actual object type + if streams[-1].type_id in delta_types: + raise BadObject(sha, "Could not resolve delta object") + return OInfo(sha, streams[-1].type, target_size) + # END handle stream + + #{ Read-Database like Interface + + def info(self, sha): + """Retrieve information about the object identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OInfo instance, with 20 byte sha""" + return self._object(sha, False) + + def stream(self, sha): + """Retrieve an object stream along with its information as identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OStream instance, with 20 byte sha""" + return self._object(sha, True) + + def info_at_index(self, index): + """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" + return self._object(None, False, index) + + def stream_at_index(self, index): + """As ``stream``, but uses a PackIndexFile compatible index to refer to the + object""" + return self._object(None, True, index) + + #} END Read-Database like Interface + + #{ Interface + + def pack(self): + """:return: the underlying pack file instance""" + return self._pack + + def index(self): + """:return: the underlying pack index file instance""" + return self._index + + def is_valid_stream(self, sha, use_crc=False): + """ + Verify that the stream at the given sha is valid. + + :param use_crc: if True, the index' crc is run over the compressed stream of + the object, which is much faster than checking the sha1. It is also + more prone to unnoticed corruption or manipulation. + :param sha: 20 byte sha1 of the object whose stream to verify + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than + just this stream. + If False, the object will be decompressed and the sha generated. It must + match the given sha + + :return: True if the stream is valid + :raise UnsupportedOperation: If the index is version 1 only + :raise BadObject: sha was not found""" + if use_crc: + if self._index.version() < 2: + raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") + # END handle index version + + index = self._sha_to_index(sha) + offset = self._index.offset(index) + next_offset = self._offset_map[offset] + crc_value = self._index.crc(index) + + # create the current crc value, on the compressed object data + # Read it in chunks, without copying the data + crc_update = zlib.crc32 + pack_data = self._pack.data() + cur_pos = offset + this_crc_value = 0 + while cur_pos < next_offset: + rbound = min(cur_pos + chunk_size, next_offset) + size = rbound - cur_pos + this_crc_value = crc_update(pack_data[cur_pos:cur_pos + size], this_crc_value) + cur_pos += size + # END window size loop + + # crc returns signed 32 bit numbers, the AND op forces it into unsigned + # mode ... wow, sneaky, from dulwich. + return (this_crc_value & 0xffffffff) == crc_value + else: + shawriter = Sha1Writer() + stream = self._object(sha, as_stream=True) + # write a loose object, which is the basis for the sha + write_object(stream.type, stream.size, stream.read, shawriter.write) + + assert shawriter.sha(as_hex=False) == sha + return shawriter.sha(as_hex=False) == sha + # END handle crc/sha verification + + def info_iter(self): + """ + :return: Iterator over all objects in this pack. The iterator yields + OInfo instances""" + return self._iter_objects(as_stream=False) + + def stream_iter(self): + """ + :return: iterator over all objects in this pack. The iterator yields + OStream instances""" + return self._iter_objects(as_stream=True) + + def collect_streams_at_offset(self, offset): + """ + As the version in the PackFile, but can resolve REF deltas within this pack + For more info, see ``collect_streams`` + + :param offset: offset into the pack file at which the object can be found""" + streams = self._pack.collect_streams(offset) + + # try to resolve the last one if needed. It is assumed to be either + # a REF delta, or a base object, as OFFSET deltas are resolved by the pack + if streams[-1].type_id == REF_DELTA: + stream = streams[-1] + while stream.type_id in delta_types: + if stream.type_id == REF_DELTA: + # smmap can return memory view objects, which can't be compared as buffers/bytes can ... + if isinstance(stream.delta_info, memoryview): + sindex = self._index.sha_to_index(stream.delta_info.tobytes()) + else: + sindex = self._index.sha_to_index(stream.delta_info) + if sindex is None: + break + stream = self._pack.stream(self._index.offset(sindex)) + streams.append(stream) + else: + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who + # would do that ;) ? We can handle it though + stream = self._pack.stream(stream.delta_info) + streams.append(stream) + # END handle ref delta + # END resolve ref streams + # END resolve streams + + return streams + + def collect_streams(self, sha): + """ + As ``PackFile.collect_streams``, but takes a sha instead of an offset. + Additionally, ref_delta streams will be resolved within this pack. + If this is not possible, the stream will be left alone, hence it is adivsed + to check for unresolved ref-deltas and resolve them before attempting to + construct a delta stream. + + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect + :return: list of streams, first being the actual object delta, the last being + a possibly unresolved base object. + :raise BadObject:""" + return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + + @classmethod + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count=None, zlib_compression=zlib.Z_BEST_SPEED): + """ + Create a new pack by putting all objects obtained by the object_iterator + into a pack which is written using the pack_write method. + The respective index is produced as well if index_write is not Non. + + :param object_iter: iterator yielding odb output objects + :param pack_write: function to receive strings to write into the pack stream + :param indx_write: if not None, the function writes the index file corresponding + to the pack. + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store + all items into a list to get the number, which uses more memory than necessary. + :param zlib_compression: the zlib compression level to use + :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack + and over all contents of the index. If index_write was None, index_binsha will be None + + **Note:** The destination of the write functions is up to the user. It could + be a socket, or a file for instance + + **Note:** writes only undeltified objects""" + objs = object_iter + if not object_count: + if not isinstance(object_iter, (tuple, list)): + objs = list(object_iter) + # END handle list type + object_count = len(objs) + # END handle object + + pack_writer = FlexibleSha1Writer(pack_write) + pwrite = pack_writer.write + ofs = 0 # current offset into the pack file + index = None + wants_index = index_write is not None + + # write header + pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) + ofs += 12 + + if wants_index: + index = IndexWriter() + # END handle index header + + actual_count = 0 + for obj in objs: + actual_count += 1 + crc = 0 + + # object header + hdr = create_pack_object_header(obj.type_id, obj.size) + if index_write: + crc = crc32(hdr) + else: + crc = None + # END handle crc + pwrite(hdr) + + # data stream + zstream = zlib.compressobj(zlib_compression) + ostream = obj.stream + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc=crc) + assert(br == obj.size) + if wants_index: + index.append(obj.binsha, crc, ofs) + # END handle index + + ofs += len(hdr) + bw + if actual_count == object_count: + break + # END abort once we are done + # END for each object + + if actual_count != object_count: + raise ValueError( + "Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + # END count assertion + + # write footer + pack_sha = pack_writer.sha(as_hex=False) + assert len(pack_sha) == 20 + pack_write(pack_sha) + ofs += len(pack_sha) # just for completeness ;) + + index_sha = None + if wants_index: + index_sha = index.write(pack_sha, index_write) + # END handle index + + return pack_sha, index_sha + + @classmethod + def create(cls, object_iter, base_dir, object_count=None, zlib_compression=zlib.Z_BEST_SPEED): + """Create a new on-disk entity comprised of a properly named pack file and a properly named + and corresponding index file. The pack contains all OStream objects contained in object iter. + :param base_dir: directory which is to contain the files + :return: PackEntity instance initialized with the new pack + + **Note:** for more information on the other parameters see the write_pack method""" + pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) + index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) + pack_write = lambda d: os.write(pack_fd, d) + index_write = lambda d: os.write(index_fd, d) + + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) + os.close(pack_fd) + os.close(index_fd) + + fmt = "pack-%s.%s" + new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) + new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) + os.rename(pack_path, new_pack_path) + os.rename(index_path, new_index_path) + + return cls(new_pack_path) + + #} END interface diff --git a/gitdb/gitdb/stream.py b/gitdb/gitdb/stream.py new file mode 100644 index 000000000..7c59f0b3c --- /dev/null +++ b/gitdb/gitdb/stream.py @@ -0,0 +1,755 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ + +from io import BytesIO + +import mmap +import os +import sys +import zlib + +from gitdb.fun import ( + msb_size, + stream_copy, + apply_delta_data, + connect_deltas, + delta_types +) + +from gitdb.util import ( + allocate_memory, + LazyMixin, + make_sha, + write, + close, +) + +from gitdb.const import NULL_BYTE, BYTE_SPACE +from gitdb.utils.encoding import force_bytes + +has_perf_mod = False +try: + from gitdb_speedups._perf import apply_delta as c_apply_delta + has_perf_mod = True +except ImportError: + pass + +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') + + +#{ RO Streams + +class DecompressMemMapReader(LazyMixin): + + """Reads data in chunks from a memory map and decompresses it. The client sees + only the uncompressed data, respective file-like read calls are handling on-demand + buffered decompression accordingly + + A constraint on the total size of bytes is activated, simulating + a logical file within a possibly larger physical memory area + + To read efficiently, you clearly don't want to read individual bytes, instead, + read a few kilobytes at least. + + **Note:** The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of + times we actually allocate. An own zlib implementation would be good here + to better support streamed reading - it would only need to keep the mmap + and decompress it into chunks, that's all ... """ + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + '_cbr', '_phi') + + max_read_size = 512 * 1024 # currently unused + + def __init__(self, m, close_on_deletion, size=None): + """Initialize with mmap for stream reading + :param m: must be content data - use new if you have object data and no size""" + self._m = m + self._zip = zlib.decompressobj() + self._buf = None # buffer of decompressed bytes + self._buflen = 0 # length of bytes in buffer + if size is not None: + self._s = size # size of uncompressed data to read in total + self._br = 0 # num uncompressed bytes read + self._cws = 0 # start byte of compression window + self._cwe = 0 # end byte of compression window + self._cbr = 0 # number of compressed bytes read + self._phi = False # is True if we parsed the header info + self._close = close_on_deletion # close the memmap on deletion ? + + def _set_cache_(self, attr): + assert attr == '_s' + # only happens for size, which is a marker to indicate we still + # have to parse the header from the stream + self._parse_header_info() + + def __del__(self): + self.close() + + def _parse_header_info(self): + """If this stream contains object data, parse the header info and skip the + stream to a point where each read will yield object content + + :return: parsed type_string, size""" + # read header + # should really be enough, cgit uses 8192 I believe + # And for good reason !! This needs to be that high for the header to be read correctly in all cases + maxb = 8192 + self._s = maxb + hdr = self.read(maxb) + hdrend = hdr.find(NULL_BYTE) + typ, size = hdr[:hdrend].split(BYTE_SPACE) + size = int(size) + self._s = size + + # adjust internal state to match actual header length that we ignore + # The buffer will be depleted first on future reads + self._br = 0 + hdrend += 1 + self._buf = BytesIO(hdr[hdrend:]) + self._buflen = len(hdr) - hdrend + + self._phi = True + + return typ, size + + #{ Interface + + @classmethod + def new(self, m, close_on_deletion=False): + """Create a new DecompressMemMapReader instance for acting as a read-only stream + This method parses the object header from m and returns the parsed + type and size, as well as the created stream instance. + + :param m: memory map on which to operate. It must be object data ( header + contents ) + :param close_on_deletion: if True, the memory map will be closed once we are + being deleted""" + inst = DecompressMemMapReader(m, close_on_deletion, 0) + typ, size = inst._parse_header_info() + return typ, size, inst + + def data(self): + """:return: random access compatible data we are working on""" + return self._m + + def close(self): + """Close our underlying stream of compressed bytes if this was allowed during initialization + :return: True if we closed the underlying stream + :note: can be called safely + """ + if self._close: + if hasattr(self._m, 'close'): + self._m.close() + self._close = False + # END handle resource freeing + + def compressed_bytes_read(self): + """ + :return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" + # ABSTRACT: When decompressing a byte stream, it can be that the first + # x bytes which were requested match the first x bytes in the loosely + # compressed datastream. This is the worst-case assumption that the reader + # does, it assumes that it will get at least X bytes from X compressed bytes + # in call cases. + # The caveat is that the object, according to our known uncompressed size, + # is already complete, but there are still some bytes left in the compressed + # stream that contribute to the amount of compressed bytes. + # How can we know that we are truly done, and have read all bytes we need + # to read ? + # Without help, we cannot know, as we need to obtain the status of the + # decompression. If it is not finished, we need to decompress more data + # until it is finished, to yield the actual number of compressed bytes + # belonging to the decompressed object + # We are using a custom zlib module for this, if its not present, + # we try to put in additional bytes up for decompression if feasible + # and check for the unused_data. + + # Only scrub the stream forward if we are officially done with the + # bytes we were to have. + if self._br == self._s and not self._zip.unused_data: + # manipulate the bytes-read to allow our own read method to continue + # but keep the window at its current position + self._br = 0 + if hasattr(self._zip, 'status'): + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop custom zlib + else: + # pass in additional pages, until we have unused data + while not self._zip.unused_data and self._cbr != len(self._m): + self.read(mmap.PAGESIZE) + # END scrub-loop default zlib + # END handle stream scrubbing + + # reset bytes read, just to be sure + self._br = self._s + # END handle stream scrubbing + + # unused data ends up in the unconsumed tail, which was removed + # from the count already + return self._cbr + + #} END interface + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + + self._zip = zlib.decompressobj() + self._br = self._cws = self._cwe = self._cbr = 0 + if self._phi: + self._phi = False + del(self._s) # trigger header parsing on first access + # END skip header + + def read(self, size=-1): + if size < 1: + size = self._s - self._br + else: + size = min(size, self._s - self._br) + # END clamp size + + if size == 0: + return b'' + # END handle depletion + + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the + # header from the zlib stream + dat = b'' + if self._buf: + if self._buflen >= size: + # have enough data + dat = self._buf.read(size) + self._buflen -= size + self._br += size + return dat + else: + dat = self._buf.read() # ouch, duplicates data + size -= self._buflen + self._br += self._buflen + + self._buflen = 0 + self._buf = None + # END handle buffer len + # END handle buffer + + # decompress some data + # Abstract: zlib needs to operate on chunks of our memory map ( which may + # be large ), as it will otherwise and always fill in the 'unconsumed_tail' + # attribute which possible reads our whole map to the end, forcing + # everything to be read from disk even though just a portion was requested. + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps + # the tail smaller than our chunk-size. This causes 'only' the chunk to be + # copied once, and another copy of a part of it when it creates the unconsumed + # tail. We have to use it to hand in the appropriate amount of bytes during + # the next read. + # + # Decompress in a loop until we have produced `size` bytes or run out + # of progress. Iteration (instead of recursion) keeps the call bounded + # for streams that consume many input bytes per produced output byte + # (e.g. zlib stored blocks of length zero); the previous recursive + # form blew the stack on inputs > ~1500 empty blocks (issue #120 + # follow-up). + dcompdat = b'' + while True: + tail = self._zip.unconsumed_tail + remaining = size - len(dcompdat) + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would save some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + remaining + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + remaining + # END handle tail + + # if window is too small, make it larger so zip can decompress something + if self._cwe - self._cws < 8: + self._cwe = self._cws + 8 + # END adjust winsize + + # takes a slice, but doesn't copy the data, it says ... + indata = self._m[self._cws:self._cwe] + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + chunk = self._zip.decompress(indata, remaining) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. + # They are thorough, and I assume it is truly working. + # Why is this logic as convoluted as it is ? Please look at the table in + # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. + # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # table in the github issue. This is it ... it was the only way I could make this work everywhere. + # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . + if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': + unused_datalen = len(self._zip.unconsumed_tail) + else: + unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) + # # end handle very special case ... + + consumed = len(indata) - unused_datalen + self._cbr += consumed + self._br += len(chunk) + if chunk: + if not isinstance(dcompdat, bytearray): + dcompdat = bytearray(dcompdat) + dcompdat.extend(chunk) + + # Stop when we have enough or there is no path to more output. + # `chunk` may legitimately be empty mid-stream when zlib is + # consuming header / dictionary frames; in that case we keep + # iterating as long as we are still feeding zlib new bytes + # (consumed > 0) and zlib has not flagged end-of-stream. The + # compressed_bytes_read() scrub loop drives this same code with + # _br manipulated to 0 past zip EOF; it terminates here because + # `getattr(_zip, 'eof', False)` is True or no compressed bytes + # are consumed. The empty-block recursion attack from issue #120 + # follow-up is bounded by the iteration; each empty block does + # consume input, so the loop walks the stream forward a constant + # amount per iteration without growing the call stack. + if len(dcompdat) >= size or self._br >= self._s: + break + zip_eof = getattr(self._zip, 'eof', False) + if not chunk and (zip_eof or len(indata) == 0 or consumed == 0): + break + # END iterative decompress + + if dat: + dcompdat = dat + dcompdat + # END prepend our cached data + + return dcompdat + + +class DeltaApplyReader(LazyMixin): + + """A reader which dynamically applies pack deltas to a base object, keeping the + memory demands to a minimum. + + The size of the final object is only obtainable once all deltas have been + applied, unless it is retrieved from a pack index. + + The uncompressed Delta has the following layout (MSB being a most significant + bit encoded dynamic size): + + * MSB Source Size - the size of the base against which the delta was created + * MSB Target Size - the size of the resulting data after the delta was applied + * A list of one byte commands (cmd) which are followed by a specific protocol: + + * cmd & 0x80 - copy delta_data[offset:offset+size] + + * Followed by an encoded offset into the delta data + * Followed by an encoded size of the chunk to copy + + * cmd & 0x7f - insert + + * insert cmd bytes from the delta buffer into the output stream + + * cmd == 0 - invalid operation ( or error in delta stream ) + """ + __slots__ = ( + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers + "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read + ) + + #{ Configuration + k_max_memory_move = 250 * 1000 * 1000 + #} END configuration + + def __init__(self, stream_list): + """Initialize this instance with a list of streams, the first stream being + the delta to apply on top of all following deltas, the last stream being the + base object onto which to apply the deltas""" + assert len(stream_list) > 1, "Need at least one delta and one base stream" + + self._bstream = stream_list[-1] + self._dstreams = tuple(stream_list[:-1]) + self._br = 0 + + def _set_cache_too_slow_without_c(self, attr): + # the direct algorithm is fastest and most direct if there is only one + # delta. Also, the extra overhead might not be worth it for items smaller + # than X - definitely the case in python, every function call costs + # huge amounts of time + # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: + if len(self._dstreams) == 1: + return self._set_cache_brute_(attr) + + # Aggregate all deltas into one delta in reverse order. Hence we take + # the last delta, and reverse-merge its ancestor delta, until we receive + # the final delta data stream. + dcl = connect_deltas(self._dstreams) + + # call len directly, as the (optional) c version doesn't implement the sequence + # protocol + if dcl.rbound() == 0: + self._size = 0 + self._mm_target = allocate_memory(0) + return + # END handle empty list + + self._size = dcl.rbound() + self._mm_target = allocate_memory(self._size) + + bbuf = allocate_memory(self._bstream.size) + stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) + + # APPLY CHUNKS + write = self._mm_target.write + dcl.apply(bbuf, write) + + self._mm_target.seek(0) + + def _set_cache_brute_(self, attr): + """If we are here, we apply the actual deltas""" + # TODO: There should be a special case if there is only one stream + # Then the default-git algorithm should perform a tad faster, as the + # delta is not peaked into, causing less overhead. + buffer_info_list = list() + max_target_size = 0 + for dstream in self._dstreams: + buf = dstream.read(512) # read the header information + X + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + buffer_info_list.append((buf[offset:], offset, src_size, target_size)) + max_target_size = max(max_target_size, target_size) + # END for each delta stream + + # sanity check - the first delta to apply should have the same source + # size as our actual base stream + base_size = self._bstream.size + target_size = max_target_size + + # if we have more than 1 delta to apply, we will swap buffers, hence we must + # assure that all buffers we use are large enough to hold all the results + if len(self._dstreams) > 1: + base_size = target_size = max(base_size, max_target_size) + # END adjust buffer sizes + + # Allocate private memory map big enough to hold the first base buffer + # We need random access to it + bbuf = allocate_memory(base_size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + + # allocate memory map large enough for the largest (intermediate) target + # We will use it as scratch space for all delta ops. If the final + # target buffer is smaller than our allocated space, we just use parts + # of it upon return. + tbuf = allocate_memory(target_size) + + # for each delta to apply, memory map the decompressed delta and + # work on the op-codes to reconstruct everything. + # For the actual copying, we use a seek and write pattern of buffer + # slices. + final_target_size = None + for (dbuf, offset, src_size, target_size), dstream in zip(reversed(buffer_info_list), reversed(self._dstreams)): + # allocate a buffer to hold all delta data - fill in the data for + # fast access. We do this as we know that reading individual bytes + # from our stream would be slower than necessary ( although possible ) + # The dbuf buffer contains commands after the first two MSB sizes, the + # offset specifies the amount of bytes read to get the sizes. + ddata = allocate_memory(dstream.size - offset) + ddata.write(dbuf) + # read the rest from the stream. The size we give is larger than necessary + stream_copy(dstream.read, ddata.write, dstream.size, 256 * mmap.PAGESIZE) + + ####################################################################### + if 'c_apply_delta' in globals(): + c_apply_delta(bbuf, ddata, tbuf) + else: + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) + ####################################################################### + + # finally, swap out source and target buffers. The target is now the + # base for the next delta to apply + bbuf, tbuf = tbuf, bbuf + bbuf.seek(0) + tbuf.seek(0) + final_target_size = target_size + # END for each delta to apply + + # its already seeked to 0, constrain it to the actual size + # NOTE: in the end of the loop, it swaps buffers, hence our target buffer + # is not tbuf, but bbuf ! + self._mm_target = bbuf + self._size = final_target_size + + #{ Configuration + if not has_perf_mod: + _set_cache_ = _set_cache_brute_ + else: + _set_cache_ = _set_cache_too_slow_without_c + + #} END configuration + + def read(self, count=0): + bl = self._size - self._br # bytes left + if count < 1 or count > bl: + count = bl + # NOTE: we could check for certain size limits, and possibly + # return buffers instead of strings to prevent byte copying + data = self._mm_target.read(count) + self._br += len(data) + return data + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self._br = 0 + self._mm_target.seek(0) + + #{ Interface + + @classmethod + def new(cls, stream_list): + """ + Convert the given list of streams into a stream which resolves deltas + when reading from it. + + :param stream_list: two or more stream objects, first stream is a Delta + to the object that you want to resolve, followed by N additional delta + streams. The list's last stream must be a non-delta stream. + + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled""" + if len(stream_list) < 2: + raise ValueError("Need at least two streams") + # END single object special handling + + if stream_list[-1].type_id in delta_types: + raise ValueError( + "Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + return cls(stream_list) + + #} END interface + + #{ OInfo like Interface + + @property + def type(self): + return self._bstream.type + + @property + def type_id(self): + return self._bstream.type_id + + @property + def size(self): + """:return: number of uncompressed bytes in the stream""" + return self._size + + #} END oinfo like interface + + +#} END RO streams + + +#{ W Streams + +class Sha1Writer: + + """Simple stream writer which produces a sha whenever you like as it degests + everything it is supposed to write""" + __slots__ = "sha1" + + def __init__(self): + self.sha1 = make_sha() + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :param data: byte object + :return: length of incoming data""" + + self.sha1.update(data) + + return len(data) + + # END stream interface + + #{ Interface + + def sha(self, as_hex=False): + """:return: sha so far + :param as_hex: if True, sha will be hex-encoded, binary otherwise""" + if as_hex: + return self.sha1.hexdigest() + return self.sha1.digest() + + #} END interface + + +class FlexibleSha1Writer(Sha1Writer): + + """Writer producing a sha1 while passing on the written bytes to the given + write function""" + __slots__ = 'writer' + + def __init__(self, writer): + Sha1Writer.__init__(self) + self.writer = writer + + def write(self, data): + Sha1Writer.write(self, data) + self.writer(data) + + +class ZippedStoreShaWriter(Sha1Writer): + + """Remembers everything someone writes to it and generates a sha""" + __slots__ = ('buf', 'zip') + + def __init__(self): + Sha1Writer.__init__(self) + self.buf = BytesIO() + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Seeking currently only supports to rewind written data + Multiple writes are not supported""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self.buf.seek(0) + + def getvalue(self): + """:return: string value from the current stream position to the end""" + return self.buf.getvalue() + + +class FDCompressedSha1Writer(Sha1Writer): + + """Digests data written to it, making the sha available, then compress the + data and write it to the file descriptor + + **Note:** operates on raw file descriptors + **Note:** for this to work, you have to use the close-method of this instance""" + __slots__ = ("fd", "sha1", "zip") + + # default exception + exc = IOError("Failed to write all bytes to filedescriptor") + + def __init__(self, fd): + super().__init__() + self.fd = fd + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: length of incoming data""" + self.sha1.update(data) + cdata = self.zip.compress(data) + bytes_written = write(self.fd, cdata) + + if bytes_written != len(cdata): + raise self.exc + + return len(data) + + def close(self): + remainder = self.zip.flush() + if write(self.fd, remainder) != len(remainder): + raise self.exc + return close(self.fd) + + #} END stream interface + + +class FDStream: + + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') + + def __init__(self, fd): + self._fd = fd + self._pos = 0 + + def write(self, data): + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos + + def close(self): + close(self._fd) + + +class NullStream: + + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) + + +#} END W streams diff --git a/gitdb/gitdb/test/__init__.py b/gitdb/gitdb/test/__init__.py new file mode 100644 index 000000000..03bd406be --- /dev/null +++ b/gitdb/gitdb/test/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/gitdb/test/db/__init__.py b/gitdb/gitdb/test/db/__init__.py new file mode 100644 index 000000000..03bd406be --- /dev/null +++ b/gitdb/gitdb/test/db/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/gitdb/test/db/lib.py b/gitdb/gitdb/test/db/lib.py new file mode 100644 index 000000000..408dd8c66 --- /dev/null +++ b/gitdb/gitdb/test/db/lib.py @@ -0,0 +1,128 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Base classes for object db testing""" +from gitdb.test.lib import ( + with_rw_directory, + with_packs_rw, + fixture_path, + TestBase +) + +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter +) + +from gitdb.base import ( + IStream, + OStream, + OInfo +) + +from gitdb.exc import BadObject +from gitdb.typ import str_blob_type + +from io import BytesIO + +from struct import pack + + +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') + + +class TestDBBase(TestBase): + + """Base class providing testing routines on databases""" + + # data + two_lines = b'1234\nhello world' + all_data = (two_lines, ) + + def _assert_object_writing_simple(self, db): + # write a bunch of objects and query their streams and info + null_objs = db.size() + ni = 250 + for i in range(ni): + data = pack(">L", i) + istream = IStream(str_blob_type, len(data), BytesIO(data)) + new_istream = db.store(istream) + assert new_istream is istream + assert db.has_object(istream.binsha) + + info = db.info(istream.binsha) + assert isinstance(info, OInfo) + assert info.type == istream.type and info.size == istream.size + + stream = db.stream(istream.binsha) + assert isinstance(stream, OStream) + assert stream.binsha == info.binsha and stream.type == info.type + assert stream.read() == data + # END for each item + + assert db.size() == null_objs + ni + shas = list(db.sha_iter()) + assert len(shas) == db.size() + assert len(shas[0]) == 20 + + def _assert_object_writing(self, db): + """General tests to verify object writing, compatible to ObjectDBW + **Note:** requires write access to the database""" + # start in 'dry-run' mode, using a simple sha1 writer + ostreams = (ZippedStoreShaWriter, None) + for ostreamcls in ostreams: + for data in self.all_data: + dry_run = ostreamcls is not None + ostream = None + if ostreamcls is not None: + ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) + # END create ostream + + prev_ostream = db.set_ostream(ostream) + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + istream = IStream(str_blob_type, len(data), BytesIO(data)) + + # store returns same istream instance, with new sha set + my_istream = db.store(istream) + sha = istream.binsha + assert my_istream is istream + assert db.has_object(sha) != dry_run + assert len(sha) == 20 + + # verify data - the slow way, we want to run code + if not dry_run: + info = db.info(sha) + assert str_blob_type == info.type + assert info.size == len(data) + + ostream = db.stream(sha) + assert ostream.read() == data + assert ostream.type == str_blob_type + assert ostream.size == len(data) + else: + self.assertRaises(BadObject, db.info, sha) + self.assertRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we passed as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.binsha is not None + prev_sha = istream.binsha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.binsha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip_best + assert ostream.getvalue() == new_ostream.getvalue() + # END for each data set + # END for each dry_run mode diff --git a/gitdb/gitdb/test/db/test_git.py b/gitdb/gitdb/test/db/test_git.py new file mode 100644 index 000000000..73ac1a08c --- /dev/null +++ b/gitdb/gitdb/test/db/test_git.py @@ -0,0 +1,53 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +import os +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) +from gitdb.exc import BadObject +from gitdb.db import GitDB +from gitdb.base import OStream, OInfo +from gitdb.util import bin_to_hex + + +class TestGitDB(TestDBBase): + + def test_reading(self): + gdb = GitDB(os.path.join(self.gitrepopath, 'objects')) + + # we have packs and loose objects, alternates doesn't necessarily exist + assert 1 < len(gdb.databases()) < 4 + + # access should be possible + gitdb_sha = next(gdb.sha_iter()) + assert isinstance(gdb.info(gitdb_sha), OInfo) + assert isinstance(gdb.stream(gitdb_sha), OStream) + ni = 50 + assert gdb.size() >= ni + sha_list = list(gdb.sha_iter()) + assert len(sha_list) == gdb.size() + sha_list = sha_list[:ni] # speed up tests ... + + # This is actually a test for compound functionality, but it doesn't + # have a separate test module + # test partial shas + # this one as uneven and quite short + gitdb_sha_hex = bin_to_hex(gitdb_sha) + assert gdb.partial_to_complete_sha_hex(gitdb_sha_hex[:5]) == gitdb_sha + + # mix even/uneven hexshas + for i, binsha in enumerate(sha_list): + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8 - (i % 2)]) == binsha + # END for each sha + + self.assertRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") + + @with_rw_directory + def test_writing(self, path): + gdb = GitDB(path) + + # its possible to write objects + self._assert_object_writing(gdb) diff --git a/gitdb/gitdb/test/db/test_loose.py b/gitdb/gitdb/test/db/test_loose.py new file mode 100644 index 000000000..295e2eecd --- /dev/null +++ b/gitdb/gitdb/test/db/test_loose.py @@ -0,0 +1,36 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) +from gitdb.db import LooseObjectDB +from gitdb.exc import BadObject +from gitdb.util import bin_to_hex + + +class TestLooseDB(TestDBBase): + + @with_rw_directory + def test_basics(self, path): + ldb = LooseObjectDB(path) + + # write data + self._assert_object_writing(ldb) + + # verify sha iteration and size + shas = list(ldb.sha_iter()) + assert shas and len(shas[0]) == 20 + + assert len(shas) == ldb.size() + + # verify find short object + long_sha = bin_to_hex(shas[-1]) + for short_sha in (long_sha[:20], long_sha[:5]): + assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha + # END for each sha + + self.assertRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + # raises if no object could be found diff --git a/gitdb/gitdb/test/db/test_mem.py b/gitdb/gitdb/test/db/test_mem.py new file mode 100644 index 000000000..882e54fe5 --- /dev/null +++ b/gitdb/gitdb/test/db/test_mem.py @@ -0,0 +1,34 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) +from gitdb.db import ( + MemoryDB, + LooseObjectDB +) + + +class TestMemoryDB(TestDBBase): + + @with_rw_directory + def test_writing(self, path): + mdb = MemoryDB() + + # write data + self._assert_object_writing_simple(mdb) + + # test stream copy + ldb = LooseObjectDB(path) + assert ldb.size() == 0 + num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) + assert num_streams_copied == mdb.size() + + assert ldb.size() == mdb.size() + for sha in mdb.sha_iter(): + assert ldb.has_object(sha) + assert ldb.stream(sha).read() == mdb.stream(sha).read() + # END verify objects where copied and are equal diff --git a/gitdb/gitdb/test/db/test_pack.py b/gitdb/gitdb/test/db/test_pack.py new file mode 100644 index 000000000..bd07906da --- /dev/null +++ b/gitdb/gitdb/test/db/test_pack.py @@ -0,0 +1,87 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, + with_packs_rw +) +from gitdb.db import PackedDB + +from gitdb.exc import BadObject, AmbiguousObjectName +from gitdb.util import mman + +import os +import random +import sys + +import pytest + +class TestPackDB(TestDBBase): + + @with_rw_directory + @with_packs_rw + def test_writing(self, path): + if sys.platform == "win32": + pytest.skip("FIXME: Currently fail on windows") + + pdb = PackedDB(path) + + # on demand, we init our pack cache + num_packs = len(pdb.entities()) + assert pdb._st_mtime != 0 + + # test pack directory changed: + # packs removed - rename a file, should affect the glob + pack_path = pdb.entities()[0].pack().path() + new_pack_path = pack_path + "renamed" + if sys.platform == "win32": + # While using this function, we are not allowed to have any handle + # to this path, which is currently not the case. The pack caching + # does still have a handle :-( + mman.force_map_handle_removal_win(pack_path) + os.rename(pack_path, new_pack_path) + + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs - 1 + + # packs added + os.rename(new_pack_path, pack_path) + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs + + # bang on the cache + # access the Entities directly, as there is no iteration interface + # yet ( or required for now ) + sha_list = list(pdb.sha_iter()) + assert len(sha_list) == pdb.size() + + # hit all packs in random order + random.shuffle(sha_list) + + for sha in sha_list: + pdb.info(sha) + pdb.stream(sha) + # END for each sha to query + + # test short finding - be a bit more brutal here + max_bytes = 19 + min_bytes = 2 + num_ambiguous = 0 + for i, sha in enumerate(sha_list): + short_sha = sha[:max((i % max_bytes), min_bytes)] + try: + assert pdb.partial_to_complete_sha(short_sha, len(short_sha) * 2) == sha + except AmbiguousObjectName: + num_ambiguous += 1 + pass # valid, we can have short objects + # END exception handling + # END for each sha to find + + # we should have at least one ambiguous, considering the small sizes + # but in our pack, there is no ambiguous ... + # assert num_ambiguous + + # non-existing + self.assertRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) diff --git a/gitdb/gitdb/test/db/test_ref.py b/gitdb/gitdb/test/db/test_ref.py new file mode 100644 index 000000000..0816e649f --- /dev/null +++ b/gitdb/gitdb/test/db/test_ref.py @@ -0,0 +1,58 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, +) +from gitdb.db import ReferenceDB + +from gitdb.util import ( + NULL_BIN_SHA, + hex_to_bin +) + +import os + + +class TestReferenceDB(TestDBBase): + + def make_alt_file(self, alt_path, alt_list): + """Create an alternates file which contains the given alternates. + The list can be empty""" + with open(alt_path, "wb") as alt_file: + for alt in alt_list: + alt_file.write(alt.encode("utf-8") + b"\n") + + @with_rw_directory + def test_writing(self, path): + alt_path = os.path.join(path, 'alternates') + rdb = ReferenceDB(alt_path) + assert len(rdb.databases()) == 0 + assert rdb.size() == 0 + assert len(list(rdb.sha_iter())) == 0 + + # try empty, non-existing + assert not rdb.has_object(NULL_BIN_SHA) + + # setup alternate file + # add two, one is invalid + own_repo_path = os.path.join(self.gitrepopath, 'objects') # use own repo + self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + # we should now find a default revision of ours + gitdb_sha = next(rdb.sha_iter()) + assert rdb.has_object(gitdb_sha) + + # remove valid + self.make_alt_file(alt_path, ["just/one/invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 0 + + # add valid + self.make_alt_file(alt_path, [own_repo_path]) + rdb.update_cache() + assert len(rdb.databases()) == 1 diff --git a/gitdb/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b b/gitdb/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b new file mode 100644 index 000000000..021c2db34 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b differ diff --git a/gitdb/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 b/gitdb/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 new file mode 100644 index 000000000..d60aeeff7 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx b/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx new file mode 100644 index 000000000..fda5969bc Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack b/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack new file mode 100644 index 000000000..a3209d2be Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx b/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx new file mode 100644 index 000000000..a7d6c7177 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack b/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack new file mode 100644 index 000000000..955c424c7 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx b/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx new file mode 100644 index 000000000..87c635f48 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack b/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack new file mode 100644 index 000000000..a69b28ac6 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack differ diff --git a/gitdb/gitdb/test/lib.py b/gitdb/gitdb/test/lib.py new file mode 100644 index 000000000..8e602342c --- /dev/null +++ b/gitdb/gitdb/test/lib.py @@ -0,0 +1,192 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Utilities used in ODB testing""" +from gitdb import OStream + +import sys +import random +from array import array + +from io import BytesIO + +import glob +import unittest +import tempfile +import shutil +import os +import gc +import logging +from functools import wraps + + +#{ Bases + +class TestBase(unittest.TestCase): + """Base class for all tests + + TestCase providing access to readonly repositories using the following member variables. + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git + """ + + #{ Invvariants + k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" + #} END invariants + + @classmethod + def setUpClass(cls): + try: + super().setUpClass() + except AttributeError: + pass + + cls.gitrepopath = os.environ.get(cls.k_env_git_repo) + if not cls.gitrepopath: + logging.info( + "You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", cls.k_env_git_repo) + ospd = os.path.dirname + cls.gitrepopath = os.path.join(ospd(ospd(ospd(__file__))), '.git') + # end assure gitrepo is set + assert cls.gitrepopath.endswith('.git') + + +#} END bases + +#{ Decorators + +def with_rw_directory(func): + """Create a temporary directory which can be written to, remove it if the + test succeeds, but leave it otherwise to aid additional debugging""" + + def wrapper(self): + path = tempfile.mktemp(prefix=func.__name__) + os.mkdir(path) + keep = False + try: + try: + return func(self, path) + except Exception: + sys.stderr.write(f"Test {type(self).__name__}.{func.__name__} failed, output is at {path!r}\n") + keep = True + raise + finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + if not keep: + gc.collect() + shutil.rmtree(path) + # END handle exception + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + + +def with_packs_rw(func): + """Function that provides a path into which the packs for testing should be + copied. Will pass on the path to the actual function afterwards""" + + def wrapper(self, path): + src_pack_glob = fixture_path('packs/*') + copy_files_globbed(src_pack_glob, path, hard_link_ok=True) + return func(self, path) + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + +#} END decorators + +#{ Routines + + +def fixture_path(relapath=''): + """:return: absolute path into the fixture directory + :param relapath: relative path into the fixtures directory, or '' + to obtain the fixture directory itself""" + return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + + +def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): + """Copy all files found according to the given source glob into the target directory + :param hard_link_ok: if True, hard links will be created if possible. Otherwise + the files will be copied""" + for src_file in glob.glob(source_glob): + if hard_link_ok and hasattr(os, 'link'): + target = os.path.join(target_dir, os.path.basename(src_file)) + try: + os.link(src_file, target) + except OSError: + shutil.copy(src_file, target_dir) + # END handle cross device links ( and resulting failure ) + else: + shutil.copy(src_file, target_dir) + # END try hard link + # END for each file to copy + + +def make_bytes(size_in_bytes, randomize=False): + """:return: string with given size in bytes + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes // 4 + producer = range(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return a.tobytes() + + +def make_object(type, data): + """:return: bytes resembling an uncompressed object""" + odata = "blob %i\0" % len(data) + return odata.encode("ascii") + data + + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), BytesIO(d) + +#} END routines + +#{ Stream Utilities + + +class DummyStream: + + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False + + def read(self, size): + self.was_read = True + self.bytes = size + + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read + + +class DeriveTest(OStream): + + def __init__(self, sha, type, size, stream, *args, **kwargs): + self.myarg = kwargs.pop('myarg') + self.args = args + + def _assert(self): + assert self.args + assert self.myarg + +#} END stream utilitiess diff --git a/gitdb/gitdb/test/performance/__init__.py b/gitdb/gitdb/test/performance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gitdb/gitdb/test/performance/lib.py b/gitdb/gitdb/test/performance/lib.py new file mode 100644 index 000000000..36916ede3 --- /dev/null +++ b/gitdb/gitdb/test/performance/lib.py @@ -0,0 +1,17 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains library functions""" +from gitdb.test.lib import TestBase + + + +#{ Base Classes + +class TestBigRepoR(TestBase): + """A placeholder in case we want to add additional functionality to all performance test-cases + """ + + +#} END base classes diff --git a/gitdb/gitdb/test/performance/test_pack.py b/gitdb/gitdb/test/performance/test_pack.py new file mode 100644 index 000000000..fc3d3349f --- /dev/null +++ b/gitdb/gitdb/test/performance/test_pack.py @@ -0,0 +1,127 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Performance tests for object store""" + +from gitdb.test.performance.lib import ( + TestBigRepoR +) + +from gitdb import ( + MemoryDB, + GitDB, + IStream, +) +from gitdb.typ import str_blob_type +from gitdb.exc import UnsupportedOperation +from gitdb.db.pack import PackedDB + +import sys +import os +from time import time + + +class TestPackedDBPerformance(TestBigRepoR): + + def test_pack_random_access(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # sha lookup + st = time() + sha_list = list(pdb.sha_iter()) + elapsed = time() - st + ns = len(sha_list) + print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / (elapsed or 1)), file=sys.stderr) + + # sha lookup: best-case and worst case access + pdb_pack_info = pdb._pack_info + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + + # discard cache + del(pdb._entities) + pdb.entities() + print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % + (ns, len(pdb.entities()), elapsed, ns / (elapsed or 1)), file=sys.stderr) + # END for each random mode + + # query info and streams only + max_items = 10000 # can wait longer when testing memory + for pdb_fun in (pdb.info, pdb.stream): + st = time() + for sha in sha_list[:max_items]: + pdb_fun(sha) + elapsed = time() - st + print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % + (max_items, pdb_fun.__name__.upper(), elapsed, max_items / (elapsed or 1)), file=sys.stderr) + # END for each function + + # retrieve stream and read all + max_items = 5000 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in sha_list[:max_items]: + stream = pdb_stream(sha) + read_len = len(stream.read()) + assert read_len == stream.size + total_size += stream.size + elapsed = time() - st + total_kib = total_size / 1000 + print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % + (max_items, total_kib, total_kib / (elapsed or 1), elapsed, max_items / (elapsed or 1)), file=sys.stderr) + + def test_loose_correctness(self): + """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back + into the loose object db (memory). + This should help finding dormant issues like this one https://github.com/gitpython-developers/GitPython/issues/220 + faster + :note: It doesn't seem this test can find the issue unless the given pack contains highly compressed + data files, like archives.""" + from gitdb.util import bin_to_hex + pdb = GitDB(os.path.join(self.gitrepopath, 'objects')) + mdb = MemoryDB() + for c, sha in enumerate(pdb.sha_iter()): + ostream = pdb.stream(sha) + # the issue only showed on larger files which are hardly compressible ... + if ostream.type != str_blob_type: + continue + istream = IStream(ostream.type, ostream.size, ostream.stream) + mdb.store(istream) + assert istream.binsha == sha, "Failed on object %s" % bin_to_hex(sha).decode('ascii') + # this can fail ... sometimes, so the packs dataset should be huge + assert len(mdb.stream(sha).read()) == ostream.size + + if c and c % 1000 == 0: + print("Verified %i loose object compression/decompression cycles" % c, file=sys.stderr) + mdb._cache.clear() + # end for each sha to copy + + def test_correctness(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + # disabled for now as it used to work perfectly, checking big repositories takes a long time + print("Endurance run: verify streaming of objects (crc and sha)", file=sys.stderr) + for crc in range(2): + count = 0 + st = time() + for entity in pdb.entities(): + pack_verify = entity.is_valid_stream + sha_by_index = entity.index().sha + for index in range(entity.index().size()): + try: + assert pack_verify(sha_by_index(index), use_crc=crc) + count += 1 + except UnsupportedOperation: + pass + # END ignore old indices + # END for each index + # END for each entity + elapsed = time() - st + print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % + (count, crc, elapsed, count / (elapsed or 1)), file=sys.stderr) + # END for each verify mode diff --git a/gitdb/gitdb/test/performance/test_pack_streaming.py b/gitdb/gitdb/test/performance/test_pack_streaming.py new file mode 100644 index 000000000..80c798bbc --- /dev/null +++ b/gitdb/gitdb/test/performance/test_pack_streaming.py @@ -0,0 +1,81 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Specific test for pack streams only""" + +from gitdb.test.performance.lib import ( + TestBigRepoR +) + +from gitdb.db.pack import PackedDB +from gitdb.stream import NullStream +from gitdb.pack import PackEntity + +import os +import sys +from time import time + + +class CountedNullStream(NullStream): + __slots__ = '_bw' + + def __init__(self): + self._bw = 0 + + def bytes_written(self): + return self._bw + + def write(self, d): + self._bw += NullStream.write(self, d) + + +class TestPackStreamingPerformance(TestBigRepoR): + + def test_pack_writing(self): + # see how fast we can write a pack from object streams. + # This will not be fast, as we take time for decompressing the streams as well + ostream = CountedNullStream() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + ni = 1000 + count = 0 + st = time() + for sha in pdb.sha_iter(): + count += 1 + pdb.stream(sha) + if count == ni: + break + # END gather objects for pack-writing + elapsed = time() - st + print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % + (ni, elapsed, ni / (elapsed or 1)), file=sys.stderr) + + st = time() + PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) + elapsed = time() - st + total_kb = ostream.bytes_written() / 1000 + print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % + (total_kb, elapsed, total_kb / (elapsed or 1)), sys.stderr) + + def test_stream_reading(self): + # raise SkipTest() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # streaming only, meant for --with-profile runs + ni = 5000 + count = 0 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + if count == ni: + break + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + count += 1 + elapsed = time() - st + total_kib = total_size / 1000 + print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % + (ni, total_kib, total_kib / (elapsed or 1), elapsed, ni / (elapsed or 1)), sys.stderr) diff --git a/gitdb/gitdb/test/performance/test_stream.py b/gitdb/gitdb/test/performance/test_stream.py new file mode 100644 index 000000000..fb10871ae --- /dev/null +++ b/gitdb/gitdb/test/performance/test_stream.py @@ -0,0 +1,106 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Performance data streaming performance""" + +from gitdb.test.performance.lib import TestBigRepoR +from gitdb.db import LooseObjectDB +from gitdb import IStream + +from gitdb.util import bin_to_hex, remove +from gitdb.fun import chunk_size + +from time import time +import os +import sys + + +from gitdb.test.lib import ( + make_memory_file, + with_rw_directory, +) + + +#{ Utilities +def read_chunked_stream(stream): + total = 0 + while True: + chunk = stream.read(chunk_size) + total += len(chunk) + if len(chunk) < chunk_size: + break + # END read stream loop + assert total == stream.size + return stream + + +#} END utilities + +class TestObjDBPerformance(TestBigRepoR): + + large_data_size_bytes = 1000 * 1000 * 50 # some MiB should do it + moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB + + @with_rw_directory + def test_large_data_streaming(self, path): + ldb = LooseObjectDB(path) + string_ios = list() # list of streams we previously created + + # serial mode + for randomize in range(2): + desc = (randomize and 'random ') or '' + print("Creating %s data ..." % desc, file=sys.stderr) + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print("Done (in %f s)" % elapsed, file=sys.stderr) + string_ios.append(stream) + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.store(IStream('blob', size, stream)).binsha + elapsed_add = time() - st + assert ldb.has_object(sha) + db_file = ldb.readable_db_object_path(bin_to_hex(sha)) + fsize_kib = os.path.getsize(db_file) / 1000 + + size_kib = size / 1000 + print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % + (size_kib, fsize_kib, desc, elapsed_add, size_kib / (elapsed_add or 1)), file=sys.stderr) + + # reading all at once + st = time() + ostream = ldb.stream(sha) + shadata = ostream.read() + elapsed_readall = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % + (size_kib, desc, elapsed_readall, size_kib / (elapsed_readall or 1)), file=sys.stderr) + + # reading in chunks of 1 MiB + cs = 512 * 1000 + chunks = list() + st = time() + ostream = ldb.stream(sha) + while True: + data = ostream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed_readchunks = time() - st + + stream.seek(0) + assert b''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % + (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / (elapsed_readchunks or 1)), file=sys.stderr) + + # del db file so we keep something to do + ostream = None # To release the file handle (win) + remove(db_file) + # END for each randomization factor diff --git a/gitdb/gitdb/test/test_base.py b/gitdb/gitdb/test/test_base.py new file mode 100644 index 000000000..17906c9c2 --- /dev/null +++ b/gitdb/gitdb/test/test_base.py @@ -0,0 +1,105 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Test for object db""" +from gitdb.test.lib import ( + TestBase, + DummyStream, + DeriveTest, +) + +from gitdb import ( + OInfo, + OPackInfo, + ODeltaPackInfo, + OStream, + OPackStream, + ODeltaPackStream, + IStream +) +from gitdb.util import ( + NULL_BIN_SHA +) + +from gitdb.typ import ( + str_blob_type +) + + +class TestBaseTypes(TestBase): + + def test_streams(self): + # test info + sha = NULL_BIN_SHA + s = 20 + blob_id = 3 + + info = OInfo(sha, str_blob_type, s) + assert info.binsha == sha + assert info.type == str_blob_type + assert info.type_id == blob_id + assert info.size == s + + # test pack info + # provides type_id + pinfo = OPackInfo(0, blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + assert pinfo.pack_offset == 0 + + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + assert dpinfo.pack_offset == 0 + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + assert dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + + # derive with own args + DeriveTest(sha, str_blob_type, s, stream, 'mine', myarg=3)._assert() + + # test istream + istream = IStream(str_blob_type, s, stream) + assert istream.binsha == None + istream.binsha = sha + assert istream.binsha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + assert istream.size == s * 2 + assert istream.type == str_blob_type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) diff --git a/gitdb/gitdb/test/test_example.py b/gitdb/gitdb/test/test_example.py new file mode 100644 index 000000000..3b4c9084b --- /dev/null +++ b/gitdb/gitdb/test/test_example.py @@ -0,0 +1,43 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module with examples from the tutorial section of the docs""" +import os +from gitdb.test.lib import TestBase +from gitdb import IStream +from gitdb.db import LooseObjectDB + +from io import BytesIO + + +class TestExamples(TestBase): + + def test_base(self): + ldb = LooseObjectDB(os.path.join(self.gitrepopath, 'objects')) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + assert ldb.has_object(oinfo.binsha) + # END for each sha in database + # assure we close all files + try: + del(ostream) + del(oinfo) + except UnboundLocalError: + pass + # END ignore exception if there are no loose objects + + data = b"my data" + istream = IStream("blob", len(data), BytesIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) diff --git a/gitdb/gitdb/test/test_pack.py b/gitdb/gitdb/test/test_pack.py new file mode 100644 index 000000000..e72348228 --- /dev/null +++ b/gitdb/gitdb/test/test_pack.py @@ -0,0 +1,249 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Test everything about packs reading and writing""" +from gitdb.test.lib import ( + TestBase, + with_rw_directory, + fixture_path +) + +from gitdb.stream import DeltaApplyReader + +from gitdb.pack import ( + PackEntity, + PackIndexFile, + PackFile +) + +from gitdb.base import ( + OInfo, + OStream, +) + +from gitdb.fun import delta_types +from gitdb.exc import UnsupportedOperation +from gitdb.util import to_bin_sha + +import pytest + +import os +import tempfile + + +#{ Utilities +def bin_sha_from_filename(filename): + return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) +#} END utilities + + +class TestPack(TestBase): + + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) + packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) + packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) + packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + packfile_v2_3_ascii = ( + fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) + + def _assert_index_file(self, index, version, size): + assert index.packfile_checksum() != index.indexfile_checksum() + assert len(index.packfile_checksum()) == 20 + assert len(index.indexfile_checksum()) == 20 + assert index.version() == version + assert index.size() == size + assert len(index.offsets()) == size + + # get all data of all objects + for oidx in range(index.size()): + sha = index.sha(oidx) + assert oidx == index.sha_to_index(sha) + + entry = index.entry(oidx) + assert len(entry) == 3 + + assert entry[0] == index.offset(oidx) + assert entry[1] == sha + assert entry[2] == index.crc(oidx) + + # verify partial sha + for l in (4, 8, 11, 17, 20): + assert index.partial_sha_to_index(sha[:l], l * 2) == oidx + + # END for each object index in indexfile + self.assertRaises(ValueError, index.partial_sha_to_index, "\0", 2) + + def _assert_pack_file(self, pack, version, size): + assert pack.version() == 2 + assert pack.size() == size + assert len(pack.checksum()) == 20 + + num_obj = 0 + for obj in pack.stream_iter(): + num_obj += 1 + info = pack.info(obj.pack_offset) + stream = pack.stream(obj.pack_offset) + + assert info.pack_offset == stream.pack_offset + assert info.type_id == stream.type_id + assert hasattr(stream, 'read') + + # it should be possible to read from both streams + assert obj.read() == stream.read() + + streams = pack.collect_streams(obj.pack_offset) + assert streams + + # read the stream + try: + dstream = DeltaApplyReader.new(streams) + except ValueError: + # ignore these, old git versions use only ref deltas, + # which we haven't resolved ( as we are without an index ) + # Also ignore non-delta streams + continue + # END get deltastream + + # read all + data = dstream.read() + assert len(data) == dstream.size + + # test seek + dstream.seek(0) + assert dstream.read() == data + + # read chunks + # NOTE: the current implementation is safe, it basically transfers + # all calls to the underlying memory map + + # END for each object + assert num_obj == size + + def test_pack_index(self): + # check version 1 and 2 + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + index = PackIndexFile(indexfile) + self._assert_index_file(index, version, size) + # END run tests + + def test_pack(self): + # there is this special version 3, but apparently its like 2 ... + for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): + pack = PackFile(packfile) + self._assert_pack_file(pack, version, size) + # END for each pack to test + + @with_rw_directory + def test_pack_entity(self, rw_dir): + pack_objs = list() + for packinfo, indexinfo in ((self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): + packfile, version, size = packinfo + indexfile, version, size = indexinfo + entity = PackEntity(packfile) + assert entity.pack().path() == packfile + assert entity.index().path() == indexfile + pack_objs.extend(entity.stream_iter()) + + count = 0 + for info, stream in zip(entity.info_iter(), entity.stream_iter()): + count += 1 + assert info.binsha == stream.binsha + assert len(info.binsha) == 20 + assert info.type_id == stream.type_id + assert info.size == stream.size + + # we return fully resolved items, which is implied by the sha centric access + assert not info.type_id in delta_types + + # try all calls + assert len(entity.collect_streams(info.binsha)) + oinfo = entity.info(info.binsha) + assert isinstance(oinfo, OInfo) + assert oinfo.binsha is not None + ostream = entity.stream(info.binsha) + assert isinstance(ostream, OStream) + assert ostream.binsha is not None + + # verify the stream + try: + assert entity.is_valid_stream(info.binsha, use_crc=True) + except UnsupportedOperation: + pass + # END ignore version issues + assert entity.is_valid_stream(info.binsha, use_crc=False) + # END for each info, stream tuple + assert count == size + + # END for each entity + + # pack writing - write all packs into one + # index path can be None + pack_path1 = tempfile.mktemp('', "pack1", rw_dir) + pack_path2 = tempfile.mktemp('', "pack2", rw_dir) + index_path = tempfile.mktemp('', 'index', rw_dir) + iteration = 0 + + def rewind_streams(): + for obj in pack_objs: + obj.stream.seek(0) + # END utility + for ppath, ipath, num_obj in zip((pack_path1, pack_path2), + (index_path, None), + (len(pack_objs), None)): + iwrite = None + if ipath: + ifile = open(ipath, 'wb') + iwrite = ifile.write + # END handle ip + + # make sure we rewind the streams ... we work on the same objects over and over again + if iteration > 0: + rewind_streams() + # END rewind streams + iteration += 1 + + with open(ppath, 'wb') as pfile: + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + assert os.path.getsize(ppath) > 100 + + # verify pack + pf = PackFile(ppath) + assert pf.size() == len(pack_objs) + assert pf.version() == PackFile.pack_version_default + assert pf.checksum() == pack_sha + pf.close() + + # verify index + if ipath is not None: + ifile.close() + assert os.path.getsize(ipath) > 100 + idx = PackIndexFile(ipath) + assert idx.version() == PackIndexFile.index_version_default + assert idx.packfile_checksum() == pack_sha + assert idx.indexfile_checksum() == index_sha + assert idx.size() == len(pack_objs) + idx.close() + # END verify files exist + # END for each packpath, indexpath pair + + # verify the packs thoroughly + rewind_streams() + entity = PackEntity.create(pack_objs, rw_dir) + count = 0 + for info in entity.info_iter(): + count += 1 + for use_crc in range(2): + assert entity.is_valid_stream(info.binsha, use_crc) + # END for each crc mode + # END for each info + assert count == len(pack_objs) + entity.close() + + def test_pack_64(self): + # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets + # of course without really needing such a huge pack + pytest.skip('not implemented') diff --git a/gitdb/gitdb/test/test_stream.py b/gitdb/gitdb/test/test_stream.py new file mode 100644 index 000000000..f36b06b96 --- /dev/null +++ b/gitdb/gitdb/test/test_stream.py @@ -0,0 +1,200 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Test for object db""" + +from gitdb.test.lib import ( + TestBase, + DummyStream, + make_bytes, + make_object, + fixture_path +) + +from gitdb import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + LooseObjectDB, + Sha1Writer, + MemoryDB, + IStream, +) +from gitdb.util import hex_to_bin + +import zlib +from gitdb.typ import ( + str_blob_type +) + +import tempfile +import os +from io import BytesIO + + +class TestStream(TestBase): + + """Test stream classes""" + + data_sizes = (15, 10000, 1000 * 1024 + 512) + + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): + """Make stream tests - the orig_stream is seekable, allowing it to be + rewound and reused + :param cdata: the data we expect to read from stream, the contents + :param rewind_stream: function called to rewind the stream to make it ready + for reuse""" + ns = 10 + assert len(cdata) > ns - 1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + + # read in small steps + ss = len(cdata) // ns + for i in range(ns): + data = stream.read(ss) + chunk = cdata[i * ss:(i + 1) * ss] + assert data == chunk + # END for each step + rest = stream.read() + if rest: + assert rest == cdata[-len(rest):] + # END handle rest + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + rewind_stream(stream) + + # read everything + rdata = stream.read() + assert rdata == cdata + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + def test_decompress_reader(self): + for close_on_deletion in range(2): + for with_size in range(2): + for ds in self.data_sizes: + cdata = make_bytes(ds, randomize=False) + + # zdata = zipped actual data + # cdata = original content data + + # create reader + if with_size: + # need object data + zdata = zlib.compress(make_object(str_blob_type, cdata)) + typ, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + assert size == len(cdata) + assert typ == str_blob_type + + # even if we don't set the size, it will be set automatically on first read + test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) + assert test_reader._s == len(cdata) + else: + # here we need content data + zdata = zlib.compress(cdata) + reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) + assert reader._s == len(cdata) + # END get reader + + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) + + # put in a dummy stream for closing + dummy = DummyStream() + reader._m = dummy + + assert not dummy.closed + del(reader) + assert dummy.closed == close_on_deletion + # END for each datasize + # END whether size should be used + # END whether stream should be closed when deleted + + def test_sha_writer(self): + writer = Sha1Writer() + assert 2 == writer.write(b"hi") + assert len(writer.sha(as_hex=1)) == 40 + assert len(writer.sha(as_hex=0)) == 20 + + # make sure it does something ;) + prev_sha = writer.sha() + writer.write(b"hi again") + assert writer.sha() != prev_sha + + def test_compressed_writer(self): + for ds in self.data_sizes: + fd, path = tempfile.mkstemp() + ostream = FDCompressedSha1Writer(fd) + data = make_bytes(ds, randomize=False) + + # for now, just a single write, code doesn't care about chunking + assert len(data) == ostream.write(data) + ostream.close() + + # its closed already + self.assertRaises(OSError, os.close, fd) + + # read everything back, compare to data we zip + fd = os.open(path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) + written_data = os.read(fd, os.path.getsize(path)) + assert len(written_data) == os.path.getsize(path) + os.close(fd) + assert written_data == zlib.compress(data, 1) # best speed + + os.remove(path) + # END for each os + + def test_decompress_reader_special_case(self): + odb = LooseObjectDB(fixture_path('objects')) + mdb = MemoryDB() + for sha in (b'888401851f15db0eed60eb1bc29dec5ddcace911', + b'7bb839852ed5e3a069966281bb08d50012fb309b',): + ostream = odb.stream(hex_to_bin(sha)) + + # if there is a bug, we will be missing one byte exactly ! + data = ostream.read() + assert len(data) == ostream.size + + # Putting it back in should yield nothing new - after all, we have + dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data))) + assert dump.hexsha == sha + # end for each loose object sha to test + + def test_decompress_reader_chunked_read_does_not_terminate_early(self): + """Regression test for #120: read(N) must not return b'' before EOF. + + zlib can consume input without producing decompressed output (e.g. + while ingesting block headers). The reader's internal recursion + previously bailed on any empty zip output, so a caller reading in + small chunks via the standard `while chunk := stream.read(N)` idiom + would terminate at the first empty chunk -- before the actual end + of the uncompressed stream. + """ + # Highly compressible data exposes the bug because each zlib chunk + # spans many uncompressed bytes -- intermediate decompress() calls + # often return empty while consuming input. + data = b"hello world! " * 1000 + zdata = zlib.compress(data) + + # Loop with a small chunk size to force many internal read/decompression + # iterations before EOF. + for chunk_size in (1, 4, 16, 64): + reader = DecompressMemMapReader( + zdata, close_on_deletion=False, size=len(data) + ) + out = bytearray() + while True: + chunk = reader.read(chunk_size) + if not chunk: + break + out.extend(chunk) + assert bytes(out) == data, ( + f"chunk_size={chunk_size}: got {len(out)}/{len(data)} bytes" + ) + assert reader._br == reader._s, ( + f"chunk_size={chunk_size}: stream stopped at " + f"{reader._br}/{reader._s}" + ) diff --git a/gitdb/gitdb/test/test_util.py b/gitdb/gitdb/test/test_util.py new file mode 100644 index 000000000..166b33c3a --- /dev/null +++ b/gitdb/gitdb/test/test_util.py @@ -0,0 +1,100 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Test for object db""" +import tempfile +import os + +from gitdb.test.lib import TestBase +from gitdb.util import ( + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA, + LockedFD +) + + +class TestUtils(TestBase): + + def test_basics(self): + assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA + assert len(to_bin_sha(NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA.encode("ascii") + + def _cmp_contents(self, file_path, data): + # raise if data from file at file_path + # does not match data string + with open(file_path, "rb") as fp: + assert fp.read() == data.encode("ascii") + + def test_lockedfd(self): + my_file = tempfile.mktemp() + orig_data = "hello" + new_data = "world" + with open(my_file, "wb") as my_file_fp: + my_file_fp.write(orig_data.encode("ascii")) + + try: + lfd = LockedFD(my_file) + lockfilepath = lfd._lockfilepath() + + # cannot end before it was started + self.assertRaises(AssertionError, lfd.rollback) + self.assertRaises(AssertionError, lfd.commit) + + # open for writing + assert not os.path.isfile(lockfilepath) + wfd = lfd.open(write=True) + assert lfd._fd is wfd + assert os.path.isfile(lockfilepath) + + # write data and fail + os.write(wfd, new_data.encode("ascii")) + lfd.rollback() + assert lfd._fd is None + self._cmp_contents(my_file, orig_data) + assert not os.path.isfile(lockfilepath) + + # additional call doesn't fail + lfd.commit() + lfd.rollback() + + # test reading + lfd = LockedFD(my_file) + rfd = lfd.open(write=False) + assert os.read(rfd, len(orig_data)) == orig_data.encode("ascii") + + assert os.path.isfile(lockfilepath) + # deletion rolls back + del(lfd) + assert not os.path.isfile(lockfilepath) + + # write data - concurrently + lfd = LockedFD(my_file) + olfd = LockedFD(my_file) + assert not os.path.isfile(lockfilepath) + wfdstream = lfd.open(write=True, stream=True) # this time as stream + assert os.path.isfile(lockfilepath) + # another one fails + self.assertRaises(IOError, olfd.open) + + wfdstream.write(new_data.encode("ascii")) + lfd.commit() + assert not os.path.isfile(lockfilepath) + self._cmp_contents(my_file, new_data) + + # could test automatic _end_writing on destruction + finally: + os.remove(my_file) + # END final cleanup + + # try non-existing file for reading + lfd = LockedFD(tempfile.mktemp()) + try: + lfd.open(write=False) + except OSError: + assert not os.path.exists(lfd._lockfilepath()) + else: + self.fail("expected OSError") + # END handle exceptions diff --git a/gitdb/gitdb/typ.py b/gitdb/gitdb/typ.py new file mode 100644 index 000000000..314db50a7 --- /dev/null +++ b/gitdb/gitdb/typ.py @@ -0,0 +1,10 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module containing information about types known to the database""" + +str_blob_type = b'blob' +str_commit_type = b'commit' +str_tree_type = b'tree' +str_tag_type = b'tag' diff --git a/gitdb/gitdb/util.py b/gitdb/gitdb/util.py new file mode 100644 index 000000000..bb6d8797a --- /dev/null +++ b/gitdb/gitdb/util.py @@ -0,0 +1,398 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +import binascii +import os +import mmap +import sys +import time +import errno + +from io import BytesIO + +from smmap import ( + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer +) + +# initialize our global memory manager instance +# Use it to free cached (and unused) resources. +mman = SlidingWindowMapManager() +# END handle mman + +import hashlib + +try: + from struct import unpack_from +except ImportError: + from struct import unpack, calcsize + __calcsize_cache = dict() + + def unpack_from(fmt, data, offset=0): + try: + size = __calcsize_cache[fmt] + except KeyError: + size = calcsize(fmt) + __calcsize_cache[fmt] = size + # END exception handling + return unpack(fmt, data[offset: offset + size]) + # END own unpack_from implementation + + +#{ Aliases + +hex_to_bin = binascii.a2b_hex +bin_to_hex = binascii.b2a_hex + +# errors +ENOENT = errno.ENOENT + +# os shortcuts +exists = os.path.exists +mkdir = os.mkdir +chmod = os.chmod +isdir = os.path.isdir +isfile = os.path.isfile +rename = os.rename +dirname = os.path.dirname +basename = os.path.basename +join = os.path.join +read = os.read +write = os.write +close = os.close +fsync = os.fsync + + +def _retry(func, *args, **kwargs): + # Wrapper around functions, that are problematic on "Windows". Sometimes + # the OS or someone else has still a handle to the file + if sys.platform == "win32": + for _ in range(10): + try: + return func(*args, **kwargs) + except Exception: + time.sleep(0.1) + return func(*args, **kwargs) + else: + return func(*args, **kwargs) + + +def remove(*args, **kwargs): + return _retry(os.remove, *args, **kwargs) + + +# Backwards compatibility imports +from gitdb.const import ( + NULL_BIN_SHA, + NULL_HEX_SHA +) + +#} END Aliases + +#{ compatibility stuff ... + + +class _RandomAccessBytesIO: + + """Wrapper to provide required functionality in case memory maps cannot or may + not be used. This is only really required in python 2.4""" + __slots__ = '_sio' + + def __init__(self, buf=''): + self._sio = BytesIO(buf) + + def __getattr__(self, attr): + return getattr(self._sio, attr) + + def __len__(self): + return len(self.getvalue()) + + def __getitem__(self, i): + return self.getvalue()[i] + + def __getslice__(self, start, end): + return self.getvalue()[start:end] + + +def byte_ord(b): + """ + Return the integer representation of the byte string. This supports Python + 3 byte arrays as well as standard strings. + """ + try: + return ord(b) + except TypeError: + return b + +#} END compatibility stuff ... + +#{ Routines + + +def make_sha(source=b''): + """A python2.4 workaround for the sha/hashlib module fiasco + + **Note** From the dulwich project """ + try: + return hashlib.sha1(source) + except NameError: + import sha + sha1 = sha.sha(source) + return sha1 + + +def allocate_memory(size): + """:return: a file-protocol accessible memory block of the given size""" + if size == 0: + return _RandomAccessBytesIO(b'') + # END handle empty chunks gracefully + + try: + return mmap.mmap(-1, size) # read-write by default + except OSError: + # setup real memory instead + # this of course may fail if the amount of memory is not available in + # one chunk - would only be the case in python 2.4, being more likely on + # 32 bit systems. + return _RandomAccessBytesIO(b"\0" * size) + # END handle memory allocation + + +def file_contents_ro(fd, stream=False, allow_mmap=True): + """:return: read-only contents of the file represented by the file descriptor fd + + :param fd: file descriptor opened for reading + :param stream: if False, random access is provided, otherwise the stream interface + is provided. + :param allow_mmap: if True, its allowed to map the contents into memory, which + allows large files to be handled and accessed efficiently. The file-descriptor + will change its position if this is False""" + try: + if allow_mmap: + # supports stream and random access + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except OSError: + # python 2.4 issue, 0 wants to be the actual size + return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) + # END handle python 2.4 + except OSError: + pass + # END exception handling + + # read manually + contents = os.read(fd, os.fstat(fd).st_size) + if stream: + return _RandomAccessBytesIO(contents) + return contents + + +def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): + """Get the file contents at filepath as fast as possible + + :return: random access compatible memory of the given filepath + :param stream: see ``file_contents_ro`` + :param allow_mmap: see ``file_contents_ro`` + :param flags: additional flags to pass to os.open + :raise OSError: If the file could not be opened + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" + fd = os.open(filepath, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) + try: + return file_contents_ro(fd, stream, allow_mmap) + finally: + close(fd) + # END assure file is closed + + +def sliding_ro_buffer(filepath, flags=0): + """ + :return: a buffer compatible object which uses our mapped memory manager internally + ready to read the whole given filepath""" + return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + + +def to_hex_sha(sha): + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + + +def to_bin_sha(sha): + if len(sha) == 20: + return sha + return hex_to_bin(sha) + + +#} END routines + + +#{ Utilities + +class LazyMixin: + + """ + Base class providing an interface to lazily retrieve attribute values upon + first access. If slots are used, memory will only be reserved once the attribute + is actually accessed and retrieved the first time. All future accesses will + return the cached value as stored in the Instance's dict or slot. + """ + + __slots__ = tuple() + + def __getattr__(self, attr): + """ + Whenever an attribute is requested that we do not know, we allow it + to be created and set. Next time the same attribute is requested, it is simply + returned from our dict/slots. """ + self._set_cache_(attr) + # will raise in case the cache was not created + return object.__getattribute__(self, attr) + + def _set_cache_(self, attr): + """ + This method should be overridden in the derived class. + It should check whether the attribute named by attr can be created + and cached. Do nothing if you do not know the attribute or call your subclass + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented + in the single attribute.""" + pass + + +class LockedFD: + + """ + This class facilitates a safe read and write operation to a file on disk. + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite + the original file. + + When reading, we obtain a lock file, but to prevent other writers from + succeeding while we are reading the file. + + This type handles error correctly in that it will assure a consistent state + on destruction. + + **note** with this setup, parallel reading is not possible""" + __slots__ = ("_filepath", '_fd', '_write') + + def __init__(self, filepath): + """Initialize an instance with the givne filepath""" + self._filepath = filepath + self._fd = None + self._write = None # if True, we write a file + + def __del__(self): + # will do nothing if the file descriptor is already closed + if self._fd is not None: + self.rollback() + + def _lockfilepath(self): + return "%s.lock" % self._filepath + + def open(self, write=False, stream=False): + """ + Open the file descriptor for reading or writing, both in binary mode. + + :param write: if True, the file descriptor will be opened for writing. Other + wise it will be opened read-only. + :param stream: if True, the file descriptor will be wrapped into a simple stream + object which supports only reading or writing + :return: fd to read from or write to. It is still maintained by this instance + and must not be closed directly + :raise IOError: if the lock could not be retrieved + :raise OSError: If the actual file could not be opened for reading + + **note** must only be called once""" + if self._write is not None: + raise AssertionError("Called %s multiple times" % self.open) + + self._write = write + + # try to open the lock file + binary = getattr(os, 'O_BINARY', 0) + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + try: + fd = os.open(self._lockfilepath(), lockmode, int("600", 8)) + if not write: + os.close(fd) + else: + self._fd = fd + # END handle file descriptor + except OSError as e: + raise OSError("Lock at %r could not be obtained" % self._lockfilepath()) from e + # END handle lock retrieval + + # open actual file if required + if self._fd is None: + # we could specify exclusive here, as we obtained the lock anyway + try: + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + except: + # assure we release our lockfile + remove(self._lockfilepath()) + raise + # END handle lockfile + # END open descriptor for reading + + if stream: + # need delayed import + from gitdb.stream import FDStream + return FDStream(self._fd) + else: + return self._fd + # END handle stream + + def commit(self): + """When done writing, call this function to commit your changes into the + actual file. + The file descriptor will be closed, and the lockfile handled. + + **Note** can be called multiple times""" + self._end_writing(successful=True) + + def rollback(self): + """Abort your operation without any changes. The file descriptor will be + closed, and the lock released. + + **Note** can be called multiple times""" + self._end_writing(successful=False) + + def _end_writing(self, successful=True): + """Handle the lock according to the write mode """ + if self._write is None: + raise AssertionError("Cannot end operation if it wasn't started yet") + + if self._fd is None: + return + + os.close(self._fd) + self._fd = None + + lockfile = self._lockfilepath() + if self._write and successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + if isfile(self._filepath): + remove(self._filepath) + # END remove if exists + # END win32 special handling + os.rename(lockfile, self._filepath) + + # assure others can at least read the file - the tmpfile left it at rw-- + # We may also write that file, on windows that boils down to a remove- + # protection as well + chmod(self._filepath, int("644", 8)) + else: + # just delete the file so far, we failed + remove(lockfile) + # END successful handling + +#} END utilities diff --git a/gitdb/gitdb/utils/__init__.py b/gitdb/gitdb/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gitdb/gitdb/utils/encoding.py b/gitdb/gitdb/utils/encoding.py new file mode 100644 index 000000000..b534ef763 --- /dev/null +++ b/gitdb/gitdb/utils/encoding.py @@ -0,0 +1,18 @@ +def force_bytes(data, encoding="utf-8"): + if isinstance(data, bytes): + return data + + if isinstance(data, str): + return data.encode(encoding) + + return data + + +def force_text(data, encoding="utf-8"): + if isinstance(data, str): + return data + + if isinstance(data, bytes): + return data.decode(encoding) + + return str(data, encoding) diff --git a/gitdb/requirements.txt b/gitdb/requirements.txt new file mode 100644 index 000000000..1b2e11db7 --- /dev/null +++ b/gitdb/requirements.txt @@ -0,0 +1 @@ +smmap>=3.0.1,<6 diff --git a/gitdb/setup.py b/gitdb/setup.py new file mode 100755 index 000000000..924f026b8 --- /dev/null +++ b/gitdb/setup.py @@ -0,0 +1,46 @@ +from setuptools import setup + +# NOTE: This is currently duplicated from the gitdb.__init__ module, because +# that's just how you write a setup.py (nobody reads this stuff out of the +# module) + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/gitdb" +version_info = (4, 0, 12) +__version__ = '.'.join(str(i) for i in version_info) + +setup( + name="gitdb", + version=__version__, + description="Git Object Database", + author=__author__, + author_email=__contact__, + url=__homepage__, + packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), + license="BSD License", + zip_safe=False, + install_requires=['smmap>=3.0.1,<6'], + long_description="""GitDB is a pure-Python git object database""", + python_requires='>=3.7', + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", + ] +) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 118e1de22..db3540857 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -1,4 +1,7 @@ #!/bin/sh +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ set -eu @@ -37,6 +40,11 @@ fi git tag __testing_point__ # The tests need a branch called master. +# +# If master is locally absent but more than one remote has it, checkout fails +# by default even if all remotes agree, and we fall back to creating it at +# HEAD. The reflog we populate below then traces HEAD's history rather than +# a remote master's, but master is reset to __testing_point__ either way. git checkout master -- || git checkout -b master # The tests need a reflog history on the master branch. @@ -47,9 +55,6 @@ git reset --hard HEAD~1 # Point the master branch where we started, so we test the correct code. git reset --hard __testing_point__ -# The tests need submodules, including a submodule with a submodule. -git submodule update --init --recursive - # The tests need some version tags. Try to get them even in forks. This fetches # other objects too. So, locally, we always do it, for a consistent experience. if ! ci || no_version_tags; then diff --git a/pyproject.toml b/pyproject.toml index 7109389d7..b7c437bf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -addopts = "--cov=git --cov-report=term --disable-warnings -ra" +addopts = "--cov=git --cov-report=term -rfEX" filterwarnings = "ignore::DeprecationWarning" python_files = "test_*.py" tmp_path_retention_policy = "failed" @@ -13,29 +13,43 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc # --cov-report term-missing # to terminal with line numbers # --cov-report html:path # html file at path # --maxfail # number of errors before giving up -# -disable-warnings # Disable pytest warnings (not codebase warnings) -# -rfE # default test summary: list fail and error +# -rfEX # test summary: list failures, errors, and unexpected passes; omit expected failures # -ra # test summary: list all non-passing (fail, error, skip, xfail, xpass) # --ignore-glob=**/gitdb/* # ignore glob paths # filterwarnings ignore::WarningType # ignores those warnings [tool.mypy] -python_version = "3.7" +files = ["git/", "test/deprecation/"] disallow_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true -# warn_unused_ignores = true +warn_unused_ignores = true # Useful in general, but especially in test/deprecation. warn_unreachable = true -show_error_codes = true implicit_reexport = true # strict = true - # TODO: Remove when 'gitdb' is fully annotated. -exclude = ["^git/ext/gitdb"] +exclude = ["^(gitdb|smmap)/"] [[tool.mypy.overrides]] -module = "gitdb.*" +module = ["gitdb", "gitdb.*"] +follow_imports = "skip" ignore_missing_imports = true +[tool.basedpyright] +typeCheckingMode = "standard" +pythonVersion = "3.7" +include = [ + "git", + "test/deprecation", +] +extraPaths = [ + "gitdb", + "smmap", +] +exclude = [ + "gitdb", + "smmap", +] + [tool.coverage.run] source = ["git"] @@ -43,7 +57,49 @@ source = ["git"] include = ["*/git/*"] omit = ["*/git/ext/*"] -[tool.black] +[tool.ruff] +target-version = "py37" line-length = 120 -target-version = ["py37"] -extend-exclude = "git/ext/gitdb" +# Exclude a variety of commonly ignored directories. +exclude = [ + "git/ext/", + "gitdb/", + "smmap/", + "build", + "dist", +] +# Enable Pyflakes `E` and `F` codes by default. +lint.select = [ + "E", + "W", # See: https://pypi.org/project/pycodestyle + "F", # See: https://pypi.org/project/pyflakes + # "I", # See: https://pypi.org/project/isort/ + # "S", # See: https://pypi.org/project/flake8-bandit + # "UP", # See: https://docs.astral.sh/ruff/rules/#pyupgrade-up +] +lint.extend-select = [ + # "A", # See: https://pypi.org/project/flake8-builtins + "B", # See: https://pypi.org/project/flake8-bugbear + "C4", # See: https://pypi.org/project/flake8-comprehensions + "TC004", # See: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ +] +lint.ignore = [ + # If it becomes necessary to ignore any rules, list them here. +] +lint.unfixable = [ + "F401", # Module imported but unused +] + +[tool.ruff.lint.per-file-ignores] +"test/**" = [ + "B018", # useless-expression +] +"fuzzing/fuzz-targets/**" = [ + "E402", # environment setup must happen before the `git` module is imported, thus cannot happen at top of file +] + + +[tool.codespell] +ignore-words-list="afile,assertIn,doesnt,gud,uptodate" +#count = true +quiet-level = 3 diff --git a/requirements-dev.txt b/requirements-dev.txt index e3030c597..066b192b8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,8 @@ --r requirements.txt --r test-requirements.txt - -# libraries for additional local testing/linting - to be added to test-requirements.txt when all pass - -flake8-type-checking;python_version>="3.8" # checks for TYPE_CHECKING only imports - -pytest-icdiff -# pytest-profiling +-r requirements.txt +-r test-requirements.txt + +# For additional local testing/linting - to be added elsewhere eventually. +ruff >=0.8 +shellcheck +pytest-icdiff +# pytest-profiling diff --git a/requirements.txt b/requirements.txt index 7159416a9..61d8403b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ gitdb>=4.0.1,<5 -typing-extensions>=3.7.4.3;python_version<"3.8" +typing-extensions>=3.10.0.2;python_version<"3.10" diff --git a/setup.py b/setup.py index 73d1ae952..0b5293685 100755 --- a/setup.py +++ b/setup.py @@ -1,23 +1,24 @@ #!/usr/bin/env python +import os +from pathlib import Path +import sys from typing import Sequence + from setuptools import setup, find_packages from setuptools.command.build_py import build_py as _build_py from setuptools.command.sdist import sdist as _sdist -import os -import sys -with open(os.path.join(os.path.dirname(__file__), "VERSION"), encoding="utf-8") as ver_file: - VERSION = ver_file.readline().strip() -with open("requirements.txt", encoding="utf-8") as reqs_file: - requirements = reqs_file.read().splitlines() +def _read_content(path: str) -> str: + return (Path(__file__).parent / path).read_text(encoding="utf-8") -with open("test-requirements.txt", encoding="utf-8") as reqs_file: - test_requirements = reqs_file.read().splitlines() -with open("README.md", encoding="utf-8") as rm_file: - long_description = rm_file.read() +version = _read_content("VERSION").strip() +requirements = _read_content("requirements.txt").splitlines() +test_requirements = _read_content("test-requirements.txt").splitlines() +doc_requirements = _read_content("doc/requirements.txt").splitlines() +long_description = _read_content("README.md") class build_py(_build_py): @@ -48,7 +49,7 @@ def _stamp_version(filename: str) -> None: with open(filename) as f: for line in f: if "__version__ =" in line: - line = line.replace('"git"', "'%s'" % VERSION) + line = line.replace('"git"', "'%s'" % version) found = True out.append(line) except OSError: @@ -64,18 +65,21 @@ def _stamp_version(filename: str) -> None: setup( name="GitPython", cmdclass={"build_py": build_py, "sdist": sdist}, - version=VERSION, + version=version, description="GitPython is a Python library used to interact with Git repositories", author="Sebastian Thiel, Michael Trier", author_email="byronimo@gmail.com, mtrier@gmail.com", license="BSD-3-Clause", url="https://github.com/gitpython-developers/GitPython", - packages=find_packages(exclude=["test", "test.*"]), + packages=find_packages(include=["git", "git.*"]), include_package_data=True, package_dir={"git": "git"}, python_requires=">=3.7", install_requires=requirements, - extras_require={"test": test_requirements}, + extras_require={ + "test": test_requirements, + "doc": doc_requirements, + }, zip_safe=False, long_description=long_description, long_description_content_type="text/markdown", @@ -91,7 +95,6 @@ def _stamp_version(filename: str) -> None: # "Development Status :: 7 - Inactive", "Environment :: Console", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Operating System :: POSIX", "Operating System :: Microsoft :: Windows", @@ -105,5 +108,8 @@ def _stamp_version(filename: str) -> None: "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", ], ) diff --git a/smmap/.coveragerc b/smmap/.coveragerc new file mode 100644 index 000000000..e61d27ba9 --- /dev/null +++ b/smmap/.coveragerc @@ -0,0 +1,10 @@ +[run] +source = smmap + +; to make nosetests happy +[report] +omit = + */yaml* + */tests/* + */python?.?/* + */site-packages/nose/* \ No newline at end of file diff --git a/smmap/.gitignore b/smmap/.gitignore new file mode 100644 index 000000000..85cc74fdf --- /dev/null +++ b/smmap/.gitignore @@ -0,0 +1,12 @@ +*.pyc +build/ +.coverage +coverage +cover/ +dist/ +MANIFEST +.tox +*.egg-info +.noseids +*.sublime-workspace +/env/ diff --git a/smmap/LICENSE b/smmap/LICENSE new file mode 100644 index 000000000..710010f1f --- /dev/null +++ b/smmap/LICENSE @@ -0,0 +1,30 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the async project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/smmap/MANIFEST.in b/smmap/MANIFEST.in new file mode 100644 index 000000000..399a207e6 --- /dev/null +++ b/smmap/MANIFEST.in @@ -0,0 +1,2 @@ +# Include the license file +include LICENSE diff --git a/smmap/Makefile b/smmap/Makefile new file mode 100644 index 000000000..ad0912ffe --- /dev/null +++ b/smmap/Makefile @@ -0,0 +1,7 @@ +.PHONY: all clean + +all: + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all + +clean: + rm -rf build/ dist/ .eggs/ .tox/ diff --git a/smmap/README.md b/smmap/README.md new file mode 100644 index 000000000..748f0f0db --- /dev/null +++ b/smmap/README.md @@ -0,0 +1,82 @@ +## Motivation + +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + + +## Limitations + +* **System resources (file-handles) are likely to be leaked!** This is due to the library authors reliance on a deterministic `__del__()` destructor. +* The memory access is read-only by design. + + +## Overview + +![Dependency packages](https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml/badge.svg) + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + + + +## Prerequisites + +* Python 3.7+ +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +## Installing smmap + +[![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) + +Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: + +```bash +$ pip install smmap +``` + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the `setup.py` script: + +```bash +$ python setup.py install +``` + +It is advised to have a look at the **Usage Guide** for a brief introduction on the different database implementations. + + + +## Homepage and Links + +The project is maintained in the GitPython repository at https://github.com/gitpython-developers/GitPython/tree/main/smmap . + +The latest source can be cloned from github as well: + +* https://github.com/gitpython-developers/GitPython.git + + +For support, please use the git-python mailing list: + +* http://groups.google.com/group/git-python + + +Issues can be filed on github: + +* https://github.com/gitpython-developers/GitPython/issues + +A link to the pypi page related to this repository: + +* https://pypi.org/project/smmap/ + + +## License Information + +*smmap* is licensed under the New BSD License. diff --git a/smmap/SECURITY.md b/smmap/SECURITY.md new file mode 100644 index 000000000..9e0c0d16a --- /dev/null +++ b/smmap/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See [GitPython](https://github.com/gitpython-developers/GitPython/blob/main/SECURITY.md). Vulnerabilities found in `smmap` can be reported there. diff --git a/smmap/doc/.gitignore b/smmap/doc/.gitignore new file mode 100644 index 000000000..32060acdd --- /dev/null +++ b/smmap/doc/.gitignore @@ -0,0 +1,2 @@ +build +*.version_info diff --git a/smmap/doc/Makefile b/smmap/doc/Makefile new file mode 100644 index 000000000..675ec2094 --- /dev/null +++ b/smmap/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/smmap.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/smmap.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/smmap/doc/make.bat b/smmap/doc/make.bat new file mode 100644 index 000000000..6900a2a46 --- /dev/null +++ b/smmap/doc/make.bat @@ -0,0 +1,113 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +set SPHINXBUILD=sphinx-build +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\smmap.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\smmap.ghc + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/smmap/doc/source/api.rst b/smmap/doc/source/api.rst new file mode 100644 index 000000000..2e2dac41a --- /dev/null +++ b/smmap/doc/source/api.rst @@ -0,0 +1,34 @@ +.. _api-label: + +############# +API Reference +############# + +*********************** +Mapped Memory Managers +*********************** + +.. automodule:: smmap.mman + :members: + :undoc-members: + +******* +Buffers +******* + +.. automodule:: smmap.buf + :members: + :undoc-members: + +********* +Utilities +********* + +.. automodule:: smmap.util + :members: + :undoc-members: + + + + + diff --git a/smmap/doc/source/changes.rst b/smmap/doc/source/changes.rst new file mode 100644 index 000000000..faed6085d --- /dev/null +++ b/smmap/doc/source/changes.rst @@ -0,0 +1,108 @@ +######### +Changelog +######### + +****** +v5.0.3 +****** + +- declare support for Python 3.13 + +For more, see https://github.com/gitpython-developers/smmap/compare/v5.0.3...v5.0.2 + +****** +v5.0.2 +****** + +- remove a usage of mktemp + +****** +v5.0.1 +****** + +- Added support for Python 3.12 + +****** +v6.0.0 +****** + +YANKED + +- Dropped support 3.6 and 3.7 +- Declared support for Python 3.11 and 3.12 + +****** +v5.0.0 +****** + +- Dropped support 3.5 +- Added support for Python 3.10 + +****** +v4.0.0 +****** + +- Dropped support for Python 2.7 and 3.4 +- Added support for Python 3.7, 3.8, and 3.9 +- Removed unused exc.MemoryManagerError and exc.RegionCollectionError + +****** +v3.0.5 +****** + +- Restored Python 2 support removed in v3.0.2 +- Changed release signature key to 27C50E7F590947D7273A741E85194C08421980C9. + See https://keybase.io/byronbates for proof of ownership. + +****** +v3.0.4 +****** + +- Signed release (with correct key this time) + +****** +v3.0.2 +****** + +- Signed release +- Switched to GitHub Actions for CI + +****** +v3.0.1 +****** +- Switched back to the smmap package name on PyPI and fixed the smmap2 mirror package + (`#44 `_) +- Fixed setup.py ``long_description`` rendering + (`#40 `_) + +********** +v0.9.0 +********** +- Fixed issue with resources never being freed as mmaps were never closed. +- Client counting is now done manually, instead of relying on pyton's reference count + +********** +v0.8.5 +********** +- Fixed Python 3.0-3.3 regression, which also causes smmap to become about 3 times slower depending on the code path. It's related to this bug (http://bugs.python.org/issue15958), which was fixed in python 3.4 + +********** +v0.8.4 +********** +- Fixed Python 3 performance regression + +********** +v0.8.3 +********** +- Cleaned up code and assured it works sufficiently well with python 3 + +********** +v0.8.1 +********** +- A single bugfix + +********** +v0.8.0 +********** + +- Initial Release diff --git a/smmap/doc/source/conf.py b/smmap/doc/source/conf.py new file mode 100644 index 000000000..55dfc5c46 --- /dev/null +++ b/smmap/doc/source/conf.py @@ -0,0 +1,194 @@ +# +# smmap documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 8 15:14:25 2011. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.append(os.path.abspath('../../')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'smmap' +copyright = '2011, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.8.0' +# The full version, including alpha/beta/rc tags. +release = '0.8.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'smmapdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'smmap.tex', 'smmap Documentation', + 'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/smmap/doc/source/index.rst b/smmap/doc/source/index.rst new file mode 100644 index 000000000..d25ef8244 --- /dev/null +++ b/smmap/doc/source/index.rst @@ -0,0 +1,26 @@ +.. smmap documentation master file, created by + sphinx-quickstart on Wed Jun 8 15:14:25 2011. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to smmap's documentation! +================================= +**smmap** is a pure python implementation of a sliding memory map to help unifying memory mapped access on 32 and 64 bit systems and to help managing resources more efficiently. + +Contents: + +.. toctree:: + :maxdepth: 2 + + intro + tutorial + api + changes + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/smmap/doc/source/intro.rst b/smmap/doc/source/intro.rst new file mode 100644 index 000000000..109fec247 --- /dev/null +++ b/smmap/doc/source/intro.rst @@ -0,0 +1,72 @@ +########### +Motivation +########### +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + +######## +Overview +######## + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + +############# +Prerequisites +############# +* Python 3.7+ +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +########### +Limitations +########### +* The memory access is read-only by design. + +################ +Installing smmap +################ +Its easiest to install smmap using the *pip* program:: + + $ pip install smmap + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +################## +Homepage and Links +################## +The project is home on github at `https://github.com/gitpython-developers/smmap `_. + +The latest source can be cloned from github as well: + + * git://github.com/gitpython-developers/smmap.git + + +For support, please use the git-python mailing list: + + * http://groups.google.com/group/git-python + + +Issues can be filed on github: + + * https://github.com/gitpython-developers/smmap/issues + +################### +License Information +################### +*smmap* is licensed under the New BSD License. + +.. _pip: http://www.pip-installer.org/en/latest/ diff --git a/smmap/doc/source/tutorial.rst b/smmap/doc/source/tutorial.rst new file mode 100644 index 000000000..917b24594 --- /dev/null +++ b/smmap/doc/source/tutorial.rst @@ -0,0 +1,118 @@ +.. _tutorial-label: + +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying classes. + +****** +Design +****** +Per application, there is *MemoryManager* which is held as static instance and used throughout the application. It can be configured to keep your resources within certain limits. + +To access mapped regions, you require a cursor. Cursors point to exactly one file and serve as handles into it. As long as it exists, the respective memory region will remain available. + +For convenience, a buffer implementation is provided which handles cursors and resource allocation behind its simple buffer like interface. + +*************** +Memory Managers +*************** +There are two types of memory managers, one uses *static* windows, the other one uses *sliding* windows. A window is a region of a file mapped into memory. Although the names might be somewhat misleading as technically windows are always static, the *sliding* version will allocate relatively small windows whereas the *static* version will always map the whole file. + +The *static* manager does nothing more than keeping a client count on the respective memory maps which always map the whole file, which allows to make some assumptions that can lead to simplified data access and increased performance, but reduces the compatibility to 32 bit systems or giant files. + +The *sliding* memory manager therefore should be the default manager when preparing an application for handling huge amounts of data on 32 bit and 64 bit platforms:: + + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + mman.num_file_handles() + mman.mapped_memory_size() + # and many more ... + + +Cursors +******* +*Cursors* are handles that point onto a window, i.e. a region of a file mapped into memory. From them you may obtain a buffer through which the data of that window can actually be accessed:: + + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + +Now you would have to write your algorithms around this interface to properly slide through huge amounts of data. + +Alternatively you can use a convenience interface. + +******* +Buffers +******* +To make first use easier, at the expense of performance, there is a Buffer implementation which uses a cursor underneath. + +With it, you can access all data in a possibly huge file without having to take care of setting the cursor to different regions yourself:: + + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + +Disadvantages +************* +Buffers cannot be used in place of strings or maps, hence you have to slice them to have valid input for the sorts of struct and zlib. A slice means a lot of data handling overhead which makes buffers slower compared to using cursors directly. + diff --git a/smmap/setup.cfg b/smmap/setup.cfg new file mode 100644 index 000000000..38c535ba5 --- /dev/null +++ b/smmap/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +exclude = .tox,.venv,build,dist,doc diff --git a/smmap/setup.py b/smmap/setup.py new file mode 100755 index 000000000..585f0e76c --- /dev/null +++ b/smmap/setup.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +import os + +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup, find_packages + +import smmap + +if os.path.exists("README.md"): + with open("README.md", encoding="utf-8") as readme: + long_description = readme.read().replace("\r\n", "\n") +else: + long_description = "See https://github.com/gitpython-developers/GitPython/tree/main/smmap" + +setup( + name="smmap", + version=smmap.__version__, + description="A pure Python implementation of a sliding window memory map manager", + author=smmap.__author__, + author_email=smmap.__contact__, + url=smmap.__homepage__, + platforms=["any"], + license="BSD-3-Clause", + packages=find_packages(), + zip_safe=True, + python_requires=">=3.7", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", + ], + long_description=long_description, + long_description_content_type='text/markdown', +) diff --git a/smmap/smmap/__init__.py b/smmap/smmap/__init__.py new file mode 100644 index 000000000..79ec0674d --- /dev/null +++ b/smmap/smmap/__init__.py @@ -0,0 +1,11 @@ +"""Initialize the smmap package""" + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/smmap" +version_info = (5, 0, 3) +__version__ = '.'.join(str(i) for i in version_info) + +# make everything available in root package for convenience +from .mman import * +from .buf import * diff --git a/smmap/smmap/buf.py b/smmap/smmap/buf.py new file mode 100644 index 000000000..731b0644b --- /dev/null +++ b/smmap/smmap/buf.py @@ -0,0 +1,143 @@ +"""Module with a simple buffer implementation using the memory manager""" +import sys + +__all__ = ["SlidingWindowMapBuffer"] + + +class SlidingWindowMapBuffer: + + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by the provided cursor. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at the offset you used during initialization or begin_access + + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) + + def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): + """Initialize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset + + def __del__(self): + self.end_access() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.end_access() + + def __len__(self): + return self._size + + def __getitem__(self, i): + if isinstance(i, slice): + return self.__getslice__(i.start or 0, i.stop or self._size) + c = self._c + assert c.is_valid() + if i < 0: + i = self._size + i + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + return c.buffer()[i - c.ofs_begin()] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxsize: + j = self._size + if j < 0: + j = self._size + j + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i - b:j - b] + else: + l = j - i # total length + ofs = i + # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower + # in the previous iteration of this code + md = list() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # can't unless we use pure bytes + if hasattr(d, 'tobytes'): + d = d.tobytes() + md.append(d) + # END while there are bytes to read + return b''.join(md) + # END fast or slow path + #{ Interface + + def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if cursor is None the existing one will be used. + :return: True if the buffer can be used""" + if cursor: + self._c = cursor + # END update our cursor + + # reuse existing cursors if possible + if self._c is not None and self._c.is_associated(): + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + # END handle size + self._size = size + # END set size + return res + # END use our cursor + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 + if self._c is not None: + self._c.unuse_region() + # END unuse region + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface diff --git a/smmap/smmap/mman.py b/smmap/smmap/mman.py new file mode 100644 index 000000000..873f687fd --- /dev/null +++ b/smmap/smmap/mman.py @@ -0,0 +1,588 @@ +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" +from .util import ( + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, +) + +import sys +from functools import reduce + +__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] +#{ Utilities + +#}END utilities + + +class WindowCursor: + + """ + Pointer into the mapped region of the memory manager, keeping the map + alive until it is destroyed and no other client uses it. + + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager + + **Note:**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" + __slots__ = ( + '_manager', # the manager keeping all file regions + '_rlist', # a regions list with regions for our file + '_region', # our current class:`MapRegion` or None + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide + ) + + def __init__(self, manager=None, regions=None): + self._manager = manager + self._rlist = regions + self._region = None + self._ofs = 0 + self._size = 0 + + def __del__(self): + self._destroy() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._destroy() + + def _destroy(self): + """Destruction code to decrement counters""" + self.unuse_region() + + if self._rlist is not None: + # Actual client count, which doesn't include the reference kept by the manager, nor ours + # as we are about to be deleted + try: + if len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._fdict.pop(self._rlist.path_or_fd()) + # END remove regions list from manager + except (TypeError, KeyError): + # sometimes, during shutdown, getrefcount is None. Its possible + # to re-import it, however, its probably better to just ignore + # this python problem (for now). + # The next step is to get rid of the error prone getrefcount altogether. + pass + # END exception handling + # END handle regions + + def _copy_from(self, rhs): + """Copy all data from rhs into this instance, handles usage count""" + self._manager = rhs._manager + self._rlist = type(rhs._rlist)(rhs._rlist) + self._region = rhs._region + self._ofs = rhs._ofs + self._size = rhs._size + + for region in self._rlist: + region.increment_client_count() + + if self._region is not None: + self._region.increment_client_count() + # END handle regions + + def __copy__(self): + """copy module interface""" + cpy = type(self)() + cpy._copy_from(self) + return cpy + + #{ Interface + def assign(self, rhs): + """Assign rhs to this instance. This is required in order to get a real copy. + Alternatively, you can copy an existing instance using the copy module""" + self._destroy() + self._copy_from(rhs) + + def use_region(self, offset=0, size=0, flags=0): + """Assure we point to a window which allows access to the given offset into the file + + :param offset: absolute offset in bytes into the file + :param size: amount of bytes to map. If 0, all available bytes will be mapped + :param flags: additional flags to be given to os.open in case a file handle is initially opened + for mapping. Has no effect if a region can actually be reused. + :return: this instance - it should be queried for whether it points to a valid memory region. + This is not the case if the mapping failed because we reached the end of the file + + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" + need_region = True + man = self._manager + fsize = self._rlist.file_size() + size = min(size or fsize, man.window_size() or fsize) # clamp size to window size + + if self._region is not None: + if self._region.includes_ofs(offset): + need_region = False + else: + self.unuse_region() + # END handle existing region + # END check existing region + + # offset too large ? + if offset >= fsize: + return self + # END handle offset + + if need_region: + self._region = man._obtain_region(self._rlist, offset, size, flags, False) + self._region.increment_client_count() + # END need region handling + + self._ofs = offset - self._region._b + self._size = min(size, self._region.ofs_end() - offset) + + return self + + def unuse_region(self): + """Unuse the current region. Does nothing if we have no current region + + **Note:** the cursor unuses the region automatically upon destruction. It is recommended + to un-use the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" + if self._region is not None: + self._region.increment_client_count(-1) + self._region = None + # note: should reset ofs and size, but we spare that for performance. Its not + # allowed to query information if we are not valid ! + + def buffer(self): + """Return a buffer object which allows access to our memory region from our offset + to the window size. Please note that it might be smaller than you requested when calling use_region() + + **Note:** You can only obtain a buffer if this instance is_valid() ! + + **Note:** buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" + return memoryview(self._region.buffer())[self._ofs:self._ofs+self._size] + + def map(self): + """ + :return: the underlying raw memory map. Please not that the offset and size is likely to be different + to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole + file in case of StaticWindowMapManager""" + return self._region.map() + + def is_valid(self): + """:return: True if we have a valid and usable region""" + return self._region is not None + + def is_associated(self): + """:return: True if we are associated with a specific file already""" + return self._rlist is not None + + def ofs_begin(self): + """:return: offset to the first byte pointed to by our cursor + + **Note:** only if is_valid() is True""" + return self._region._b + self._ofs + + def ofs_end(self): + """:return: offset to one past the last available byte""" + # unroll method calls for performance ! + return self._region._b + self._ofs + self._size + + def size(self): + """:return: amount of bytes we point to""" + return self._size + + def region(self): + """:return: our mapped region, or None if nothing is mapped yet + :raise AssertionError: if we have no current region. This is only useful for debugging""" + return self._region + + def includes_ofs(self, ofs): + """:return: True if the given absolute offset is contained in the cursors + current region + + **Note:** cursor must be valid for this to work""" + # unroll methods + return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) + + def file_size(self): + """:return: size of the underlying file""" + return self._rlist.file_size() + + def path_or_fd(self): + """:return: path or file descriptor of the underlying mapped file""" + return self._rlist.path_or_fd() + + def path(self): + """:return: path of the underlying mapped file + :raise ValueError: if attached path is not a path""" + if isinstance(self._rlist.path_or_fd(), int): + raise ValueError("Path queried although mapping was applied to a file descriptor") + # END handle type + return self._rlist.path_or_fd() + + def fd(self): + """:return: file descriptor used to create the underlying mapping. + + **Note:** it is not required to be valid anymore + :raise ValueError: if the mapping was not created by a file descriptor""" + if isinstance(self._rlist.path_or_fd(), str): + raise ValueError("File descriptor queried although mapping was generated from path") + # END handle type + return self._rlist.path_or_fd() + + #} END interface + + +class StaticWindowMapManager: + + """Provides a manager which will produce single size cursors that are allowed + to always map the whole file. + + Clients must be written to specifically know that they are accessing their data + through a StaticWindowMapManager, as they otherwise have to deal with their window size. + + These clients would have to use a SlidingWindowMapBuffer to hide this fact. + + This type will always use a maximum window size, and optimize certain methods to + accommodate this fact""" + + __slots__ = [ + '_fdict', # mapping of path -> StorageHelper (of some kind + '_window_size', # maximum size of a window + '_max_memory_size', # maximum amount of memory we may allocate + '_max_handle_count', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] + + #{ Configuration + MapRegionListCls = MapRegionList + MapWindowCls = MapWindow + MapRegionCls = MapRegion + WindowCursorCls = WindowCursor + #} END configuration + + _MB_in_bytes = 1024 * 1024 + + def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsize): + """initialize the manager with the given parameters. + :param window_size: if -1, a default window size will be chosen depending on + the operating system's architecture. It will internally be quantified to a multiple of the page size + If 0, the window may have any size, which basically results in mapping the whole file at one + :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. + If 0, a viable default will be set depending on the system's architecture. + It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate + :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, + the manager will free as many handles as possible""" + self._fdict = dict() + self._window_size = window_size + self._max_memory_size = max_memory_size + self._max_handle_count = max_open_handles + self._memory_size = 0 + self._handle_count = 0 + + if window_size < 0: + coeff = 64 + if is_64_bit(): + coeff = 1024 + # END handle arch + self._window_size = coeff * self._MB_in_bytes + # END handle max window size + + if max_memory_size == 0: + coeff = 1024 + if is_64_bit(): + coeff = 8192 + # END handle arch + self._max_memory_size = coeff * self._MB_in_bytes + # END handle max memory size + + #{ Internal Methods + + def _collect_lru_region(self, size): + """Unmap the region which was least-recently used and has no client + :param size: size of the region we want to map next (assuming its not already mapped partially or full + if 0, we try to free any available region + :return: Amount of freed regions + + .. Note:: + We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. + + .. TODO:: + implement a case where all unusued regions are discarded efficiently. + Currently its only brute force + """ + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + lru_region = None + lru_list = None + for regions in self._fdict.values(): + for region in regions: + # check client count - if it's 1, it's just us + if (region.client_count() == 1 and + (lru_region is None or region._uc < lru_region._uc)): + lru_region = region + lru_list = regions + # END update lru_region + # END for each region + # END for each regions list + + if lru_region is None: + break + # END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + lru_region.increment_client_count(-1) + self._memory_size -= lru_region.size() + self._handle_count -= 1 + # END while there is more memory to free + return num_found + + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utility to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + if self._memory_size + size > self._max_memory_size: + self._collect_lru_region(size) + # END handle collection + + r = None + if a: + assert len(a) == 1 + r = a[0] + else: + try: + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxsize, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + # END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + # END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.append(r) + # END handle array + + assert r.includes_ofs(offset) + return r + + #}END internal methods + + #{ Interface + def make_cursor(self, path_or_fd): + """ + :return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + + **Note:** if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + + **Note:** file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. + + **Note:** Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" + regions = self._fdict.get(path_or_fd) + if regions is None: + regions = self.MapRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions + # END obtain region for path + return self.WindowCursorCls(self, regions) + + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x, y: x + y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_file_handles(self): + """:return: maximum amount of handles we may have opened""" + return self._max_handle_count + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + + **Note:** does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + # END early bailout + + num_closed = 0 + for path, rlist in self._fdict.items(): + if path.startswith(base_path): + for region in rlist: + region.release() + num_closed += 1 + # END path matches + # END for each path + return num_closed + #} END special purpose interface + + +class SlidingWindowMapManager(StaticWindowMapManager): + + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + **Note:** currently not thread-safe ! + + **Note:** in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + + def __init__(self, window_size=-1, max_memory_size=0, max_open_handles=sys.maxsize): + """Adjusts the default window size to -1""" + super().__init__(window_size, max_memory_size, max_open_handles) + + def _obtain_region(self, a, offset, size, flags, is_recursive): + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. + r = None + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo + hi) // 2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + r = a[mid] + break + # END have region + lo = mid + 1 + else: + hi = mid + # END handle position + # END while bisecting + + if r is None: + window_size = self._window_size + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(a.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + # Save calls ! + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + # END handle collection + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(a) + if len_regions == 1: + if a[0]._b <= offset: + insert_pos = 1 + # END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(a): + if region._b > offset: + insert_pos = i + break + # END if insert position is correct + # END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right side + else: + if insert_pos != len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right window + left = self.MapWindowCls.from_region(a[insert_pos - 1]) + # END adjust surrounding windows + + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + # END readjust size + + # insert new region at the right offset to keep the order + try: + if self._handle_count >= self._max_handle_count: + raise Exception + # END assert own imposed max file handles + r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + # END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + # END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.insert(insert_pos, r) + # END create new region + return r diff --git a/smmap/smmap/test/__init__.py b/smmap/smmap/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/smmap/smmap/test/lib.py b/smmap/smmap/test/lib.py new file mode 100644 index 000000000..b15b0ec6a --- /dev/null +++ b/smmap/smmap/test/lib.py @@ -0,0 +1,72 @@ +"""Provide base classes for the test system""" +from unittest import TestCase +import os +import tempfile + +__all__ = ['TestBase', 'FileCreator'] + + +#{ Utilities + +class FileCreator: + + """A instance which creates a temporary file with a prefix and a given size + and provides this info to the user. + Once it gets deleted, it will remove the temporary file as well.""" + __slots__ = ("_size", "_path") + + def __init__(self, size, prefix=''): + assert size, "Require size to be larger 0" + + self._size = size + + with tempfile.NamedTemporaryFile("wb", prefix=prefix, delete=False) as file: + self._path = file.name + file.seek(size - 1) + file.write(b'1') + + assert os.path.getsize(self.path) == size + + def __del__(self): + try: + os.remove(self.path) + except OSError: + pass + # END exception handling + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.__del__() + + @property + def path(self): + return self._path + + @property + def size(self): + return self._size + +#} END utilities + + +class TestBase(TestCase): + + """Foundation used by all tests""" + + #{ Configuration + k_window_test_size = 1000 * 1000 * 8 + 5195 + #} END configuration + + #{ Overrides + @classmethod + def setUpAll(cls): + # nothing for now + pass + + # END overrides + + #{ Interface + + #} END interface diff --git a/smmap/smmap/test/test_buf.py b/smmap/smmap/test/test_buf.py new file mode 100644 index 000000000..f0a86fb64 --- /dev/null +++ b/smmap/smmap/test/test_buf.py @@ -0,0 +1,126 @@ +from .lib import TestBase, FileCreator + +from smmap.mman import ( + SlidingWindowMapManager, + StaticWindowMapManager +) +from smmap.buf import SlidingWindowMapBuffer + +from random import randint +from time import time +import sys +import os + + +man_optimal = SlidingWindowMapManager() +man_worst_case = SlidingWindowMapManager( + window_size=TestBase.k_window_test_size // 100, + max_memory_size=TestBase.k_window_test_size // 3, + max_open_handles=15) +static_man = StaticWindowMapManager() + + +class TestBuf(TestBase): + + def test_basics(self): + with FileCreator(self.k_window_test_size, "buffer_test") as fc: + + # invalid paths fail upon construction + c = man_optimal.make_cursor(fc.path) + self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + + buf = SlidingWindowMapBuffer() # can create uninitialized buffers + assert buf.cursor() is None + + # can call end access any time + buf.end_access() + buf.end_access() + assert len(buf) == 0 + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset + assert buf.cursor().is_valid() + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert len(buf) == fc.size + assert buf.cursor().is_valid() + + # simple access + with open(fc.path, 'rb') as fp: + data = fp.read() + assert data[offset] == buf[0] + assert data[offset:offset * 2] == buf[0:offset] + + # negative indices, partial slices + assert buf[-1] == buf[len(buf) - 1] + assert buf[-10:] == buf[len(buf) - 10:len(buf)] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + del(c) + + assert man_optimal.num_file_handles() == 1 + + # PERFORMANCE + # blast away with random access and a full mapping - we don't want to + # exaggerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 100 + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ((man_optimal, 'optimal'), + (man_worst_case, 'worst case'), + (static_man, 'static optimal')): + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + del d + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + # END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000 * 1000) + mode_str = (access_mode and "slice") or "single byte" + print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), + file=sys.stderr) + # END handle access mode + del buf + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/smmap/test/test_mman.py b/smmap/smmap/test/test_mman.py new file mode 100644 index 000000000..7a5f4092c --- /dev/null +++ b/smmap/smmap/test/test_mman.py @@ -0,0 +1,224 @@ +from .lib import TestBase, FileCreator + +from smmap.mman import ( + WindowCursor, + SlidingWindowMapManager, + StaticWindowMapManager +) +from smmap.util import align_to_mmap + +from random import randint +from time import time +import os +import sys +from copy import copy + + +class TestMMan(TestBase): + + def test_cursor(self): + with FileCreator(self.k_window_test_size, "cursor_test") as fc: + man = SlidingWindowMapManager() + ci = WindowCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated() # but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path + + # copy module + cio = copy(cv) + assert not cio.is_valid() and cio.is_associated() + + # assign method + assert not ci.is_associated() + ci.assign(cv) + assert not ci.is_valid() and ci.is_associated() + + # unuse non-existing region is fine + cv.unuse_region() + cv.unuse_region() + + # destruction is fine (even multiple times) + cv._destroy() + WindowCursor(man)._destroy() + + def test_memory_manager(self): + slide_man = SlidingWindowMapManager() + static_man = StaticWindowMapManager() + + for man in (static_man, slide_man): + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + winsize_cmp_val = 0 + if isinstance(man, StaticWindowMapManager): + winsize_cmp_val = -1 + # END handle window size + assert man.window_size() > winsize_cmp_val + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + + # collection doesn't raise in 'any' mode + man._collect_lru_region(0) + # doesn't raise if we are within the limit + man._collect_lru_region(10) + + # doesn't fail if we over-allocate + assert man._collect_lru_region(sys.maxsize) == 0 + + # use a region, verify most basic functionality + with FileCreator(self.k_window_test_size, "manager_test") as fc: + fd = os.open(fc.path, os.O_RDONLY) + try: + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + with open(fc.path, 'rb') as fp: + assert c.buffer()[:] == fp.read(20)[10:] + + if isinstance(item, int): + self.assertRaises(ValueError, c.path) + else: + self.assertRaises(ValueError, c.fd) + # END handle value error + # END for each input + finally: + os.close(fd) + # END for each manasger type + + def test_memman_operation(self): + # test more access, force it to actually unmap regions + with FileCreator(self.k_window_test_size, "manager_operation_test") as fc: + with open(fc.path, 'rb') as fp: + data = fp.read() + fd = os.open(fc.path, os.O_RDONLY) + try: + max_num_handles = 15 + # small_size = + for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() // 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region() + assert rr.client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr.size() + + # assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr.ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + # but isn't larger than the max window (aligned) + assert rr.size() == align_to_mmap(man.window_size(), True) + else: + assert rr.size() == fc.size + # END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region() == rr + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region() is not rr # old region is still available, but has not cursor ref anymore + assert rr.client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + # END ignore static managers which only have one handle per file + rr = c.region() + assert rr.client_count() == 2 # manager + cursor + assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr.ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + # END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + # END statistics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset + csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset + csize - 1) + assert not includes_ofs(base_offset + csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000 * 1000) + print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), + file=sys.stderr) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + # END for each item + # END for each manager type + finally: + os.close(fd) diff --git a/smmap/smmap/test/test_tutorial.py b/smmap/smmap/test/test_tutorial.py new file mode 100644 index 000000000..31c272abb --- /dev/null +++ b/smmap/smmap/test/test_tutorial.py @@ -0,0 +1,75 @@ +from .lib import TestBase + + +class TestTutorial(TestBase): + + def test_example(self): + # Memory Managers + ################## + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + assert mman.num_file_handles() == 0 + assert mman.mapped_memory_size() == 0 + # and many more ... + + # Cursors + ########## + import smmap.test.lib + with smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") as fc: + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size() - 1] # last byte + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:] # access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset diff --git a/smmap/smmap/test/test_util.py b/smmap/smmap/test/test_util.py new file mode 100644 index 000000000..e6ac10f35 --- /dev/null +++ b/smmap/smmap/test/test_util.py @@ -0,0 +1,105 @@ +from .lib import TestBase, FileCreator + +from smmap.util import ( + MapWindow, + MapRegion, + MapRegionList, + ALLOCATIONGRANULARITY, + is_64_bit, + align_to_mmap +) + +import os +import sys + + +class TestMMan(TestBase): + + def test_window(self): + wl = MapWindow(0, 1) # left + wc = MapWindow(1, 1) # center + wc2 = MapWindow(10, 5) # another center + wr = MapWindow(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxsize) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxsize) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) + + def test_region(self): + with FileCreator(self.k_window_test_size, "window_test") as fc: + half_size = fc.size // 2 + rofs = align_to_mmap(4200, False) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) + + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) + + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 1, "no auto-counting" + + # window constructor + w = MapWindow.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + + def test_region_list(self): + with FileCreator(100, "sample_file") as fc: + fd = os.open(fc.path, os.O_RDONLY) + try: + for item in (fc.path, fd): + ml = MapRegionList(item) + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + finally: + os.close(fd) + + def test_util(self): + assert isinstance(is_64_bit(), bool) # just call it + assert align_to_mmap(1, False) == 0 + assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY diff --git a/smmap/smmap/util.py b/smmap/smmap/util.py new file mode 100644 index 000000000..fbb387211 --- /dev/null +++ b/smmap/smmap/util.py @@ -0,0 +1,222 @@ +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" +import os +import sys + +from mmap import mmap, ACCESS_READ +from mmap import ALLOCATIONGRANULARITY + +__all__ = ["align_to_mmap", "is_64_bit", + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] + +#{ Utilities + + +def align_to_mmap(num, round_up): + """ + Align the given integer number to the closest page offset, which usually is 4096 bytes. + + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY + if round_up and (res != num): + res += ALLOCATIONGRANULARITY + # END handle size + return res + + +def is_64_bit(): + """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" + return sys.maxsize > (1 << 32) - 1 + +#}END utilities + + +#{ Utility Classes + +class MapWindow: + + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) + + def __init__(self, offset, size): + self.ofs = offset + self.size = size + + def __repr__(self): + return "MapWindow(%i, %i)" % (self.ofs, self.size) + + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region._b, region.size()) + + def ofs_end(self): + return self.ofs + self.size + + def align(self): + """Assures the previous window area is contained in the new one""" + nofs = align_to_mmap(self.ofs, 0) + self.size += self.ofs - nofs # keep size constant + self.ofs = nofs + self.size = align_to_mmap(self.size, 1) + + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs -= rofs + self.size += rofs + + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + + +class MapRegion: + + """Defines a mapped region of memory, aligned to pagesizes + + **Note:** deallocates used region automatically on destruction""" + __slots__ = [ + '_b', # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_size', # cached size of our memory map + '__weakref__' + ] + + #{ Configuration + #} END configuration + + def __init__(self, path_or_fd, ofs, size, flags=0): + """Initialize a region, allocate the memory map + :param path_or_fd: path to the file to map, or the opened file descriptor + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :param flags: additional flags to be given when opening the file. + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._size = 0 + self._uc = 0 + + if isinstance(path_or_fd, int): + fd = path_or_fd + else: + fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) + # END handle fd + + try: + kwargs = dict(access=ACCESS_READ, offset=ofs) + corrected_size = size + sizeofs = ofs + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) + self._mf = mmap(fd, actual_size, **kwargs) + # END handle memory mode + + self._size = len(self._mf) + finally: + if isinstance(path_or_fd, str): + os.close(fd) + # END only close it if we opened it + # END close file handle + # We assume the first one to use us keeps us around + self.increment_client_count() + + def __repr__(self): + return "MapRegion<%i, %i>" % (self._b, self.size()) + + #{ Interface + + def buffer(self): + """:return: a buffer containing the memory""" + return self._mf + + def map(self): + """:return: a memory map containing the memory""" + return self._mf + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return self._size + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self._size + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return self._b <= ofs < self._b + self._size + + def client_count(self): + """:return: number of clients currently using this region""" + return self._uc + + def increment_client_count(self, ofs = 1): + """Adjust the usage count by the given positive or negative offset. + If usage count equals 0, we will auto-release our resources + :return: True if we released resources, False otherwise. In the latter case, we can still be used""" + self._uc += ofs + assert self._uc > -1, "Increments must match decrements, usage counter negative: %i" % self._uc + + if self.client_count() == 0: + self.release() + return True + else: + return False + # end handle release + + def release(self): + """Release all resources this instance might hold. Must only be called if there usage_count() is zero""" + self._mf.close() + + #} END interface + + +class MapRegionList(list): + + """List of MapRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path_or_fd', # path or file descriptor which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super().__new__(cls) + + def __init__(self, path_or_fd): + self._path_or_fd = path_or_fd + self._file_size = None + + def path_or_fd(self): + """:return: path or file descriptor we are attached to""" + return self._path_or_fd + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + if isinstance(self._path_or_fd, str): + self._file_size = os.stat(self._path_or_fd).st_size + else: + self._file_size = os.fstat(self._path_or_fd).st_size + # END handle path type + # END update file size + return self._file_size + +#} END utility classes diff --git a/smmap/tox.ini b/smmap/tox.ini new file mode 100644 index 000000000..f7205c5a3 --- /dev/null +++ b/smmap/tox.ini @@ -0,0 +1,17 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = flake8, py{37, 38, 39, 310, 311, 312} + +[testenv] +commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} +deps = + pytest + pytest-cov + +[testenv:flake8] +commands = flake8 {posargs} +deps = flake8 diff --git a/test-requirements.txt b/test-requirements.txt index fcdc93c1d..e2443825c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,12 +1,12 @@ -black coverage[toml] +basedpyright==1.39.9 ; python_version >= "3.9" and sys_platform != "cygwin" ddt >= 1.1.1, != 1.4.3 mock ; python_version < "3.8" -mypy +mypy==1.18.2 ; python_version >= "3.9" # pin mypy version to avoid new errors pre-commit pytest >= 7.3.1 pytest-cov pytest-instafail pytest-mock pytest-sugar -sumtypes +typing-extensions ; python_version < "3.11" diff --git a/test/deprecation/__init__.py b/test/deprecation/__init__.py new file mode 100644 index 000000000..fec3126d2 --- /dev/null +++ b/test/deprecation/__init__.py @@ -0,0 +1,19 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests of deprecation warnings and possible related attribute bugs. + +Most deprecation warnings are "basic" in the sense that there is no special complexity +to consider, in introducing them. However, to issue deprecation warnings on mere +attribute access can involve adding new dynamic behavior. This can lead to subtle bugs +or less useful dynamic metadata. It can also weaken static typing, as happens if a type +checker sees a method like ``__getattr__`` in a module or class whose attributes it did +not already judge to be dynamic. This test.deprecation submodule covers all three cases: +the basic cases, subtle dynamic behavior, and subtle static type checking issues. + +Static type checking is "tested" by a combination of code that should not be treated as +a type error but would be in the presence of particular bugs, and code that *should* be +treated as a type error and is accordingly marked ``# type: ignore[REASON]`` (for +specific ``REASON``. The latter will only produce mypy errors when the expectation is +not met if it is configured with ``warn_unused_ignores = true``. +""" diff --git a/test/deprecation/lib.py b/test/deprecation/lib.py new file mode 100644 index 000000000..9fe623a3a --- /dev/null +++ b/test/deprecation/lib.py @@ -0,0 +1,27 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Support library for deprecation tests.""" + +__all__ = ["assert_no_deprecation_warning", "suppress_deprecation_warning"] + +import contextlib +import warnings + +from typing import Generator + + +@contextlib.contextmanager +def assert_no_deprecation_warning() -> Generator[None, None, None]: + """Context manager to assert that code does not issue any deprecation warnings.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + warnings.simplefilter("error", PendingDeprecationWarning) + yield + + +@contextlib.contextmanager +def suppress_deprecation_warning() -> Generator[None, None, None]: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + yield diff --git a/test/deprecation/test_basic.py b/test/deprecation/test_basic.py new file mode 100644 index 000000000..3bf0287c7 --- /dev/null +++ b/test/deprecation/test_basic.py @@ -0,0 +1,137 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests of assorted deprecation warnings when there are no extra subtleties to check. + +This tests deprecation warnings where all that needs be verified is that a deprecated +property, function, or class issues a DeprecationWarning when used and, if applicable, +that recommended alternatives do not issue the warning. + +This is in contrast to other modules within test.deprecation, which test warnings where +there is a risk of breaking other runtime behavior, or of breaking static type checking +or making it less useful, by introducing the warning or in plausible future changes to +how the warning is implemented. That happens when it is necessary to customize attribute +access on a module or class, in a way it was not customized before, to issue a warning. +It is inapplicable to the deprecations whose warnings are tested in this module. +""" + +import pytest + +from git.diff import NULL_TREE +from git.objects.util import Traversable +from git.repo import Repo +from git.util import Iterable as _Iterable, IterableObj + +from .lib import assert_no_deprecation_warning + +# typing ----------------------------------------------------------------- + +from typing import Generator, TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + from git.diff import Diff, DiffIndex + from git.objects.commit import Commit + +# ------------------------------------------------------------------------ + + +@pytest.fixture +def commit(tmp_path: "Path") -> Generator["Commit", None, None]: + """Fixture to supply a one-commit repo's commit, enough for deprecation tests.""" + (tmp_path / "a.txt").write_text("hello\n", encoding="utf-8") + repo = Repo.init(tmp_path) + repo.index.add(["a.txt"]) + yield repo.index.commit("Initial commit") + repo.close() + + +@pytest.fixture +def diff(commit: "Commit") -> Generator["Diff", None, None]: + """Fixture to supply a single-file diff.""" + (diff,) = commit.diff(NULL_TREE) # Exactly one file in the diff. + yield diff + + +@pytest.fixture +def diffs(commit: "Commit") -> Generator["DiffIndex", None, None]: + """Fixture to supply a DiffIndex.""" + yield commit.diff(NULL_TREE) + + +def test_diff_renamed_warns(diff: "Diff") -> None: + """The deprecated Diff.renamed property issues a deprecation warning.""" + with pytest.deprecated_call(): + diff.renamed + + +def test_diff_renamed_file_does_not_warn(diff: "Diff") -> None: + """The preferred Diff.renamed_file property issues no deprecation warning.""" + with assert_no_deprecation_warning(): + diff.renamed_file + + +def test_commit_trailers_warns(commit: "Commit") -> None: + """The deprecated Commit.trailers property issues a deprecation warning.""" + with pytest.deprecated_call(): + commit.trailers + + +def test_commit_trailers_list_does_not_warn(commit: "Commit") -> None: + """The nondeprecated Commit.trailers_list property issues no deprecation warning.""" + with assert_no_deprecation_warning(): + commit.trailers_list + + +def test_commit_trailers_dict_does_not_warn(commit: "Commit") -> None: + """The nondeprecated Commit.trailers_dict property issues no deprecation warning.""" + with assert_no_deprecation_warning(): + commit.trailers_dict + + +def test_traverse_list_traverse_in_base_class_warns(commit: "Commit") -> None: + """Traversable.list_traverse's base implementation issues a deprecation warning.""" + with pytest.deprecated_call(): + Traversable.list_traverse(commit) + + +def test_traversable_list_traverse_override_does_not_warn(commit: "Commit") -> None: + """Calling list_traverse on concrete subclasses is not deprecated, does not warn.""" + with assert_no_deprecation_warning(): + commit.list_traverse() + + +def test_traverse_traverse_in_base_class_warns(commit: "Commit") -> None: + """Traversable.traverse's base implementation issues a deprecation warning.""" + with pytest.deprecated_call(): + Traversable.traverse(commit) + + +def test_traverse_traverse_override_does_not_warn(commit: "Commit") -> None: + """Calling traverse on concrete subclasses is not deprecated, does not warn.""" + with assert_no_deprecation_warning(): + commit.traverse() + + +def test_iterable_inheriting_warns() -> None: + """Subclassing the deprecated git.util.Iterable issues a deprecation warning.""" + with pytest.deprecated_call(): + + class Derived(_Iterable): + pass + + +def test_iterable_obj_inheriting_does_not_warn() -> None: + """Subclassing git.util.IterableObj is not deprecated, does not warn.""" + with assert_no_deprecation_warning(): + + class Derived(IterableObj): + pass + + +def test_diff_iter_change_type(diffs: "DiffIndex") -> None: + """The internal DiffIndex.iter_change_type function issues no deprecation warning.""" + with assert_no_deprecation_warning(): + for change_type in diffs.change_type: + [*diffs.iter_change_type(change_type=change_type)] diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py new file mode 100644 index 000000000..e44490273 --- /dev/null +++ b/test/deprecation/test_cmd_git.py @@ -0,0 +1,391 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests for dynamic and static characteristics of Git class and instance attributes. + +Currently this all relates to the deprecated :attr:`Git.USE_SHELL` class attribute, +which can also be accessed through instances. Some tests directly verify its behavior, +including deprecation warnings, while others verify that other aspects of attribute +access are not inadvertently broken by mechanisms introduced to issue the warnings. + +A note on multiprocessing +========================= + +Because USE_SHELL has no instance state, this module does not include tests of pickling +and multiprocessing: + +- Just as with a simple class attribute, when a class attribute with custom logic is set + to another value, even before a worker process is created that uses the class, the + worker process may see either the initial or new value, depending on the process start + method. With "fork", changes are preserved. With "spawn" or "forkserver", re-importing + the modules causes initial values to be set. Then the value in the parent at the time + it dispatches the task is only set in the children if the parent has the task set it, + or if it is set as a side effect of importing needed modules, or of unpickling objects + passed to the child (for example, if it is set in a top-level statement of the module + that defines the function submitted for the child worker process to call). + +- When an attribute gains new logic provided by a property or custom descriptor, and the + attribute involves instance-level state, incomplete or corrupted pickling can break + multiprocessing. (For example, when an instance attribute is reimplemented using a + descriptor that stores data in a global WeakKeyDictionary, pickled instances should be + tested to ensure they are still working correctly.) But nothing like that applies + here, because instance state is not involved. Although the situation is inherently + complex as described above, it is independent of the attribute implementation. + +- That USE_SHELL cannot be set on instances, and that when retrieved on instances it + always gives the same value as on the class, is covered in the tests here. + +A note on metaclass conflicts +============================= + +The most important DeprecationWarning is for code like ``Git.USE_SHELL = True``, which +is a security risk. But this warning may not be possible to implement without a custom +metaclass. This is because a descriptor in a class can customize all forms of attribute +access on its instances, but can only customize getting an attribute on the class. +Retrieving a descriptor from a class calls its ``__get__`` method (if defined), but +replacing or deleting it does not call its ``__set__`` or ``__delete__`` methods. + +Adding a metaclass is a potentially breaking change. This is because derived classes +that use an unrelated metaclass, whether directly or by inheriting from a class such as +abc.ABC that uses one, will raise TypeError when defined. These would have to be +modified to use a newly introduced metaclass that is a lower bound of both. Subclasses +remain unbroken in the far more typical case that they use no custom metaclass. + +The tests in this module do not establish whether the danger of setting Git.USE_SHELL to +True is high enough, and applications of deriving from Git and using an unrelated custom +metaclass marginal enough, to justify introducing a metaclass to issue the warnings. +""" + +import logging +import sys +from typing import Generator +import unittest.mock + +if sys.version_info >= (3, 11): + from typing import assert_type +else: + from typing_extensions import assert_type + +import pytest +from pytest import WarningsRecorder + +from git.cmd import Git, GitMeta + +from .lib import assert_no_deprecation_warning, suppress_deprecation_warning + +_USE_SHELL_DEPRECATED_FRAGMENT = "Git.USE_SHELL is deprecated" +"""Text contained in all USE_SHELL deprecation warnings, and starting most of them.""" + +_USE_SHELL_DANGEROUS_FRAGMENT = "Setting Git.USE_SHELL to True is unsafe and insecure" +"""Beginning text of USE_SHELL deprecation warnings when USE_SHELL is set True.""" + +_logger = logging.getLogger(__name__) + + +@pytest.fixture +def restore_use_shell_state() -> Generator[None, None, None]: + """Fixture to attempt to restore state associated with the USE_SHELL attribute. + + This is used to decrease the likelihood of state changes leaking out and affecting + other tests. But the goal is not to assert implementation details of USE_SHELL. + + This covers two of the common implementation strategies, for convenience in testing + both. USE_SHELL could be implemented in the metaclass: + + * With a separate _USE_SHELL backing attribute. If using a property or other + descriptor, this is the natural way to do it, but custom __getattribute__ and + __setattr__ logic, if it does more than adding warnings, may also use that. + * Like a simple attribute, using USE_SHELL itself, stored as usual in the class + dictionary, with custom __getattribute__/__setattr__ logic only to warn. + + This tries to save private state, tries to save the public attribute value, yields + to the test case, tries to restore the public attribute value, then tries to restore + private state. The idea is that if the getting or setting logic is wrong in the code + under test, the state will still most likely be reset successfully. + """ + no_value = object() + + # Try to save the original private state. + try: + old_private_value = Git._USE_SHELL # type: ignore[attr-defined] + except AttributeError: + separate_backing_attribute = False + try: + old_private_value = type.__getattribute__(Git, "USE_SHELL") + except AttributeError: + old_private_value = no_value + _logger.error("Cannot retrieve old private _USE_SHELL or USE_SHELL value") + else: + separate_backing_attribute = True + + try: + # Try to save the original public value. Rather than attempt to restore a state + # where the attribute is not set, if we cannot do this we allow AttributeError + # to propagate out of the fixture, erroring the test case before its code runs. + with suppress_deprecation_warning(): + old_public_value = Git.USE_SHELL + + # This doesn't have its own try-finally because pytest catches exceptions raised + # during the yield. (The outer try-finally catches exceptions in this fixture.) + yield + + # Try to restore the original public value. + with suppress_deprecation_warning(): + Git.USE_SHELL = old_public_value + finally: + # Try to restore the original private state. + if separate_backing_attribute: + Git._USE_SHELL = old_private_value # type: ignore[attr-defined] + elif old_private_value is not no_value: + type.__setattr__(Git, "USE_SHELL", old_private_value) + + +def test_cannot_access_undefined_on_git_class() -> None: + """Accessing a bogus attribute on the Git class remains a dynamic and static error. + + This differs from Git instances, where most attribute names will dynamically + synthesize a "bound method" that runs a git subcommand when called. + """ + with pytest.raises(AttributeError): + Git.foo # type: ignore[attr-defined] + + +def test_get_use_shell_on_class_default() -> None: + """USE_SHELL can be read as a class attribute, defaulting to False and warning.""" + with pytest.deprecated_call() as ctx: + use_shell = Git.USE_SHELL + + (message,) = [str(entry.message) for entry in ctx] # Exactly one warning. + assert message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + + assert_type(use_shell, bool) + + # This comes after the static assertion, just in case it would affect the inference. + assert not use_shell + + +def test_get_use_shell_on_instance_default() -> None: + """USE_SHELL can be read as an instance attribute, defaulting to False and warning. + + This is the same as test_get_use_shell_on_class_default above, but for instances. + The test is repeated, instead of using parametrization, for clearer static analysis. + """ + instance = Git() + + with pytest.deprecated_call() as ctx: + use_shell = instance.USE_SHELL + + (message,) = [str(entry.message) for entry in ctx] # Exactly one warning. + assert message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + + assert_type(use_shell, bool) + + # This comes after the static assertion, just in case it would affect the inference. + assert not use_shell + + +def _assert_use_shell_full_results( + set_value: bool, + reset_value: bool, + setting: WarningsRecorder, + checking: WarningsRecorder, + resetting: WarningsRecorder, + rechecking: WarningsRecorder, +) -> None: + # The attribute should take on the values set to it. + assert set_value is True + assert reset_value is False + + # Each access should warn exactly once. + (set_message,) = [str(entry.message) for entry in setting] + (check_message,) = [str(entry.message) for entry in checking] + (reset_message,) = [str(entry.message) for entry in resetting] + (recheck_message,) = [str(entry.message) for entry in rechecking] + + # Setting it to True should produce the special warning for that. + assert _USE_SHELL_DEPRECATED_FRAGMENT in set_message + assert set_message.startswith(_USE_SHELL_DANGEROUS_FRAGMENT) + + # All other operations should produce a usual warning. + assert check_message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + assert reset_message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + assert recheck_message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + + +def test_use_shell_set_and_get_on_class(restore_use_shell_state: None) -> None: + """USE_SHELL can be set and re-read as a class attribute, always warning.""" + with pytest.deprecated_call() as setting: + Git.USE_SHELL = True + with pytest.deprecated_call() as checking: + set_value = Git.USE_SHELL + with pytest.deprecated_call() as resetting: + Git.USE_SHELL = False + with pytest.deprecated_call() as rechecking: + reset_value = Git.USE_SHELL + + _assert_use_shell_full_results( + set_value, + reset_value, + setting, + checking, + resetting, + rechecking, + ) + + +def test_use_shell_set_on_class_get_on_instance(restore_use_shell_state: None) -> None: + """USE_SHELL can be set on the class and read on an instance, always warning. + + This is like test_use_shell_set_and_get_on_class but it performs reads on an + instance. There is some redundancy here in assertions about warnings when the + attribute is set, but it is a separate test so that any bugs where a read on the + class (or an instance) is needed first before a read on an instance (or the class) + are detected. + """ + instance = Git() + + with pytest.deprecated_call() as setting: + Git.USE_SHELL = True + with pytest.deprecated_call() as checking: + set_value = instance.USE_SHELL + with pytest.deprecated_call() as resetting: + Git.USE_SHELL = False + with pytest.deprecated_call() as rechecking: + reset_value = instance.USE_SHELL + + _assert_use_shell_full_results( + set_value, + reset_value, + setting, + checking, + resetting, + rechecking, + ) + + +@pytest.mark.parametrize("value", [False, True]) +def test_use_shell_cannot_set_on_instance( + value: bool, + restore_use_shell_state: None, # In case of a bug where it does set USE_SHELL. +) -> None: + instance = Git() + with pytest.raises(AttributeError): + instance.USE_SHELL = value # type: ignore[misc] # Name not in __slots__. + + +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +@pytest.mark.parametrize("original_value", [False, True]) +def test_use_shell_is_mock_patchable_on_class_as_object_attribute( + original_value: bool, + restore_use_shell_state: None, +) -> None: + """Asymmetric patching looking up USE_SHELL in ``__dict__`` doesn't corrupt state. + + Code using GitPython may temporarily set Git.USE_SHELL to a different value. Ideally + it does not use unittest.mock.patch to do so, because that makes subtle assumptions + about the relationship between attributes and dictionaries. If the attribute can be + retrieved from the ``__dict__`` rather than directly, that value is assumed the + correct one to restore, even by a normal setattr. + + The effect is that some ways of simulating a class attribute with added behavior can + cause a descriptor, such as a property, to be set as the value of its own backing + attribute during unpatching; then subsequent reads raise RecursionError. This + happens if both (a) setting it on the class is customized in a metaclass and (b) + getting it on instances is customized with a descriptor (such as a property) in the + class itself. + + Although ideally code outside GitPython would not rely on being able to patch + Git.USE_SHELL with unittest.mock.patch, the technique is widespread. Thus, USE_SHELL + should be implemented in some way compatible with it. This test checks for that. + """ + Git.USE_SHELL = original_value + if Git.USE_SHELL is not original_value: + raise RuntimeError("Can't set up the test") + new_value = not original_value + + with unittest.mock.patch.object(Git, "USE_SHELL", new_value): + assert Git.USE_SHELL is new_value + + assert Git.USE_SHELL is original_value + + +def test_execute_without_shell_arg_does_not_warn() -> None: + """No deprecation warning is issued from operations implemented using Git.execute(). + + When no ``shell`` argument is passed to Git.execute, which is when the value of + USE_SHELL is to be used, the way Git.execute itself accesses USE_SHELL does not + issue a deprecation warning. + """ + with assert_no_deprecation_warning(): + Git().version() + + +_EXPECTED_DIR_SUBSET = { + "cat_file_all", + "cat_file_header", + "GIT_PYTHON_TRACE", + "USE_SHELL", # The attribute we get deprecation warnings for. + "GIT_PYTHON_GIT_EXECUTABLE", + "refresh", + "is_cygwin", + "polish_url", + "check_unsafe_protocols", + "check_unsafe_options", + "AutoInterrupt", + "CatFileContentStream", + "__init__", + "__getattr__", + "set_persistent_git_options", + "working_dir", + "version_info", + "execute", + "environment", + "update_environment", + "custom_environment", + "transform_kwarg", + "transform_kwargs", + "__call__", + "_call_process", # Not currently considered public, but unlikely to change. + "get_object_header", + "get_object_data", + "stream_object_data", + "clear_cache", +} +"""Some stable attributes dir() should include on the Git class and its instances. + +This is intentionally incomplete, but includes substantial variety. Most importantly, it +includes both ``USE_SHELL`` and a wide sampling of other attributes. +""" + + +def test_class_dir() -> None: + """dir() on the Git class includes its statically known attributes. + + This tests that the mechanism that adds dynamic behavior to USE_SHELL accesses so + that all accesses issue warnings does not break dir() for the class, neither for + USE_SHELL nor for ordinary (non-deprecated) attributes. + """ + actual = set(dir(Git)) + assert _EXPECTED_DIR_SUBSET <= actual + + +def test_instance_dir() -> None: + """dir() on Git objects includes its statically known attributes. + + This is like test_class_dir, but for Git instances rather than the class itself. + """ + instance = Git() + actual = set(dir(instance)) + assert _EXPECTED_DIR_SUBSET <= actual + + +def test_metaclass_alias() -> None: + """GitMeta aliases Git's metaclass, whether that is type or a custom metaclass.""" + + def accept_metaclass_instance(cls: GitMeta) -> None: + """Check that cls is statically recognizable as an instance of GitMeta.""" + + accept_metaclass_instance(Git) # assert_type would expect Type[Git], not GitMeta. + + # This comes after the static check, just in case it would affect the inference. + assert type(Git) is GitMeta diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py new file mode 100644 index 000000000..2d7805e61 --- /dev/null +++ b/test/deprecation/test_compat.py @@ -0,0 +1,84 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests for dynamic and static characteristics of git.compat module attributes. + +These tests verify that the is_ attributes are available, and are even listed +in the output of dir(), but issue warnings, and that bogus (misspelled or unrecognized) +attribute access is still an error both at runtime and with mypy. This is similar to +some of the tests in test_toplevel, but the situation being tested here is simpler +because it does not involve unintuitive module aliasing or import behavior. So this only +tests attribute access, not "from" imports (whose behavior can be intuitively inferred). +""" + +import os +import sys + +if sys.version_info >= (3, 11): + from typing import assert_type +else: + from typing_extensions import assert_type + +import pytest + +import git.compat + +_MESSAGE_LEADER = "{} and other is_ aliases are deprecated." +"""Form taken by the beginning of the warnings issued for is_ access.""" + + +def test_cannot_access_undefined() -> None: + """Accessing a bogus attribute in git.compat remains a dynamic and static error.""" + with pytest.raises(AttributeError): + git.compat.foo # type: ignore[attr-defined] + + +def test_is_platform() -> None: + """The is_ attributes work, warn, and mypy accepts code accessing them.""" + fully_qualified_names = [ + "git.compat.is_win", + "git.compat.is_posix", + "git.compat.is_darwin", + ] + + with pytest.deprecated_call() as ctx: + is_win = git.compat.is_win + is_posix = git.compat.is_posix + is_darwin = git.compat.is_darwin + + assert_type(is_win, bool) + assert_type(is_posix, bool) + assert_type(is_darwin, bool) + + messages = [str(entry.message) for entry in ctx] + assert len(messages) == 3 + + for fullname, message in zip(fully_qualified_names, messages): + assert message.startswith(_MESSAGE_LEADER.format(fullname)) + + # These assertions exactly reproduce the expressions in the code under test, so they + # are not good for testing that the values are correct. Instead, their purpose is to + # ensure that any dynamic machinery put in place in git.compat to cause warnings to + # be issued does not get in the way of the intended values being accessed. + assert is_win == (os.name == "nt") + assert is_posix == (os.name == "posix") + assert is_darwin == (sys.platform == "darwin") + + +def test_dir() -> None: + """dir() on git.compat includes all public attributes, even if deprecated. + + As dir() usually does, it also has nonpublic attributes, which should also not be + removed by a custom __dir__ function, but those are less important to test. + """ + expected_subset = { + "is_win", + "is_posix", + "is_darwin", + "defenc", + "safe_decode", + "safe_encode", + "win_encode", + } + actual = set(dir(git.compat)) + assert expected_subset <= actual diff --git a/test/deprecation/test_remote_progress.py b/test/deprecation/test_remote_progress.py new file mode 100644 index 000000000..a65739d31 --- /dev/null +++ b/test/deprecation/test_remote_progress.py @@ -0,0 +1,19 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests for static characteristics of remote progress helpers.""" + +from typing import AnyStr + +from git import RemoteProgress + + +def test_parse_progress_line_override_can_return_non_none() -> None: + """Subclass overrides may return a value without violating base typing.""" + + class ReturningProgress(RemoteProgress): + def _parse_progress_line(self, line: AnyStr) -> str: + super()._parse_progress_line(line) + return "handled" + + assert ReturningProgress()._parse_progress_line("unmatched progress") == "handled" diff --git a/test/deprecation/test_toplevel.py b/test/deprecation/test_toplevel.py new file mode 100644 index 000000000..740408193 --- /dev/null +++ b/test/deprecation/test_toplevel.py @@ -0,0 +1,233 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests for dynamic and static characteristics of top-level git module attributes. + +Provided mypy has ``warn_unused_ignores = true`` set, running mypy on these test cases +checks static typing of the code under test. This is the reason for the many separate +single-line attr-defined suppressions, so those should not be replaced with a smaller +number of more broadly scoped suppressions, even where it is feasible to do so. + +Running pytest checks dynamic behavior as usual. +""" + +import itertools +import sys +from typing import Type + +if sys.version_info >= (3, 11): + from typing import assert_type +else: + from typing_extensions import assert_type + +import pytest + +import git +import git.index.base +import git.index.fun +import git.index.typ +import git.refs.head +import git.refs.log +import git.refs.reference +import git.refs.symbolic +import git.refs.tag + + +def test_cannot_access_undefined() -> None: + """Accessing a bogus attribute in git remains a dynamic and static error.""" + with pytest.raises(AttributeError): + git.foo # type: ignore[attr-defined] + + +def test_cannot_import_undefined() -> None: + """Importing a bogus attribute from git remains a dynamic and static error.""" + with pytest.raises(ImportError): + from git import foo # type: ignore[attr-defined] # noqa: F401 + + +def test_util_alias_access() -> None: + """Accessing util in git works, warns, and mypy verifies it and its attributes.""" + # The attribute access should succeed. + with pytest.deprecated_call() as ctx: + util = git.util + + # There should be exactly one warning and it should have our util-specific message. + (message,) = [str(entry.message) for entry in ctx] + assert "git.util" in message + assert "git.index.util" in message + assert "should not be relied on" in message + + # We check access through the util alias to the TemporaryFileSwap member, since it + # is slightly simpler to validate and reason about than the other public members, + # which are functions (specifically, higher-order functions for use as decorators). + from git.index.util import TemporaryFileSwap + + assert_type(util.TemporaryFileSwap, Type[TemporaryFileSwap]) + + # This comes after the static assertion, just in case it would affect the inference. + assert util.TemporaryFileSwap is TemporaryFileSwap + + +def test_util_alias_import() -> None: + """Importing util from git works, warns, and mypy verifies it and its attributes.""" + # The import should succeed. + with pytest.deprecated_call() as ctx: + from git import util + + # There may be multiple warnings. In CPython there will be currently always be + # exactly two, possibly due to the equivalent of calling hasattr to do a pre-check + # prior to retrieving the attribute for actual use. However, all warnings should + # have the same message, and it should be our util-specific message. + (message,) = {str(entry.message) for entry in ctx} + assert "git.util" in message, "Has alias." + assert "git.index.util" in message, "Has target." + assert "should not be relied on" in message, "Distinct from other messages." + + # As above, we check access through the util alias to the TemporaryFileSwap member. + from git.index.util import TemporaryFileSwap + + assert_type(util.TemporaryFileSwap, Type[TemporaryFileSwap]) + + # This comes after the static assertion, just in case it would affect the inference. + assert util.TemporaryFileSwap is TemporaryFileSwap + + +_PRIVATE_MODULE_ALIAS_TARGETS = ( + git.refs.head, + git.refs.log, + git.refs.reference, + git.refs.symbolic, + git.refs.tag, + git.index.base, + git.index.fun, + git.index.typ, +) +"""Targets of private aliases in the git module to some modules, not including util.""" + + +_PRIVATE_MODULE_ALIAS_TARGET_NAMES = ( + "git.refs.head", + "git.refs.log", + "git.refs.reference", + "git.refs.symbolic", + "git.refs.tag", + "git.index.base", + "git.index.fun", + "git.index.typ", +) +"""Expected ``__name__`` attributes of targets of private aliases in the git module.""" + + +def test_alias_target_module_names_are_by_location() -> None: + """The aliases are weird, but their targets are normal, even in ``__name__``.""" + actual = [module.__name__ for module in _PRIVATE_MODULE_ALIAS_TARGETS] + expected = list(_PRIVATE_MODULE_ALIAS_TARGET_NAMES) + assert actual == expected + + +def test_private_module_alias_access() -> None: + """Non-util private alias access works but warns and is a deliberate mypy error.""" + with pytest.deprecated_call() as ctx: + assert ( + git.head, # type: ignore[attr-defined] + git.log, # type: ignore[attr-defined] + git.reference, # type: ignore[attr-defined] + git.symbolic, # type: ignore[attr-defined] + git.tag, # type: ignore[attr-defined] + git.base, # type: ignore[attr-defined] + git.fun, # type: ignore[attr-defined] + git.typ, # type: ignore[attr-defined] + ) == _PRIVATE_MODULE_ALIAS_TARGETS + + # Each should have warned exactly once, and note what to use instead. + messages = [str(w.message) for w in ctx] + + assert len(messages) == len(_PRIVATE_MODULE_ALIAS_TARGETS) + + for fullname, message in zip(_PRIVATE_MODULE_ALIAS_TARGET_NAMES, messages): + assert message.endswith(f"Use {fullname} instead.") + + +def test_private_module_alias_import() -> None: + """Non-util private alias import works but warns and is a deliberate mypy error.""" + with pytest.deprecated_call() as ctx: + from git import head # type: ignore[attr-defined] + from git import log # type: ignore[attr-defined] + from git import reference # type: ignore[attr-defined] + from git import symbolic # type: ignore[attr-defined] + from git import tag # type: ignore[attr-defined] + from git import base # type: ignore[attr-defined] + from git import fun # type: ignore[attr-defined] + from git import typ # type: ignore[attr-defined] + + assert ( + head, + log, + reference, + symbolic, + tag, + base, + fun, + typ, + ) == _PRIVATE_MODULE_ALIAS_TARGETS + + # Each import may warn multiple times. In CPython there will be currently always be + # exactly two warnings per import, possibly due to the equivalent of calling hasattr + # to do a pre-check prior to retrieving the attribute for actual use. However, for + # each import, all messages should be the same and should note what to use instead. + messages_with_duplicates = [str(w.message) for w in ctx] + messages = [message for message, _ in itertools.groupby(messages_with_duplicates)] + + assert len(messages) == len(_PRIVATE_MODULE_ALIAS_TARGETS) + + for fullname, message in zip(_PRIVATE_MODULE_ALIAS_TARGET_NAMES, messages): + assert message.endswith(f"Use {fullname} instead.") + + +def test_dir_contains_public_attributes() -> None: + """All public attributes of the git module are present when dir() is called on it. + + This is naturally the case, but some ways of adding dynamic attribute access + behavior can change it, especially if __dir__ is defined but care is not taken to + preserve the contents that should already be present. + + Note that dir() should usually automatically list non-public attributes if they are + actually "physically" present as well, so the approach taken here to test it should + not be reproduced if __dir__ is added (instead, a call to globals() could be used, + as its keys list the automatic values). + """ + expected_subset = set(git.__all__) + actual = set(dir(git)) + assert expected_subset <= actual + + +def test_dir_does_not_contain_util() -> None: + """The util attribute is absent from the dir() of git. + + Because this behavior is less confusing than including it, where its meaning would + be assumed by users examining the dir() for what is available. + """ + assert "util" not in dir(git) + + +def test_dir_does_not_contain_private_module_aliases() -> None: + """Names from inside index and refs only pretend to be there and are not in dir(). + + The reason for omitting these is not that they are private, since private members + are usually included in dir() when actually present. Instead, these are only sort + of even there, no longer being imported and only being resolved dynamically for the + time being. In addition, it would be confusing to list these because doing so would + obscure the module structure of GitPython. + """ + expected_absent = { + "head", + "log", + "reference", + "symbolic", + "tag", + "base", + "fun", + "typ", + } + actual = set(dir(git)) + assert not (expected_absent & actual), "They should be completely disjoint." diff --git a/test/deprecation/test_types.py b/test/deprecation/test_types.py new file mode 100644 index 000000000..d3c6af645 --- /dev/null +++ b/test/deprecation/test_types.py @@ -0,0 +1,69 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests for dynamic and static characteristics of git.types module attributes.""" + +import sys + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +import pytest + +import git.types + + +def test_cannot_access_undefined() -> None: + """Accessing a bogus attribute in git.types remains a dynamic and static error.""" + with pytest.raises(AttributeError): + git.types.foo # type: ignore[attr-defined] + + +def test_can_access_lit_commit_ish_but_it_is_not_usable() -> None: + """Lit_commit_ish_can be accessed, but warns and is an invalid type annotation.""" + # It would be fine to test attribute access rather than a "from" import. But a + # "from" import is more likely to appear in actual usage, so it is used here. + with pytest.deprecated_call() as ctx: + from git.types import Lit_commit_ish + + # As noted in test_toplevel.test_util_alias_import, there may be multiple warnings, + # but all with the same message. + (message,) = {str(entry.message) for entry in ctx} + assert "Lit_commit_ish is deprecated." in message + assert 'Literal["commit", "tag", "blob", "tree"]' in message, "Has old definition." + assert 'Literal["commit", "tag"]' in message, "Has new definition." + assert "GitObjectTypeString" in message, "Has new type name for old definition." + + _: Lit_commit_ish = "commit" + + # It should be as documented (even though deliberately unusable in static checks). + assert Lit_commit_ish == Literal["commit", "tag"] + + +def test_dir() -> None: + """dir() on git.types includes public names, even ``Lit_commit_ish``. + + It also contains private names that we don't test. See test_compat.test_dir. + """ + expected_subset = { + "PathLike", + "TBD", + "AnyGitObject", + "Tree_ish", + "Commit_ish", + "GitObjectTypeString", + "Lit_commit_ish", + "Lit_config_levels", + "ConfigLevels_Tup", + "CallableProgress", + "assert_never", + "Files_TD", + "Total_TD", + "HSH_TD", + "Has_Repo", + "Has_id_attribute", + } + actual = set(dir(git.types)) + assert expected_subset <= actual diff --git a/test/fixtures/diff_numstat b/test/fixtures/diff_numstat index 44c6ca2d5..b76e467eb 100644 --- a/test/fixtures/diff_numstat +++ b/test/fixtures/diff_numstat @@ -1,2 +1,3 @@ -29 18 a.txt -0 5 b.txt +M 29 18 a.txt +M 0 5 b.txt +A 7 0 c.txt \ No newline at end of file diff --git a/test/fixtures/git_config_with_empty_quotes b/test/fixtures/git_config_with_empty_quotes new file mode 100644 index 000000000..f11fe4248 --- /dev/null +++ b/test/fixtures/git_config_with_empty_quotes @@ -0,0 +1,2 @@ +[core] + filemode = "" diff --git a/test/fixtures/git_config_with_extra_whitespace b/test/fixtures/git_config_with_extra_whitespace new file mode 100644 index 000000000..0f727cb5d --- /dev/null +++ b/test/fixtures/git_config_with_extra_whitespace @@ -0,0 +1,2 @@ +[init] + defaultBranch = trunk diff --git a/test/fixtures/git_config_with_quotes b/test/fixtures/git_config_with_quotes new file mode 100644 index 000000000..40e6710d9 --- /dev/null +++ b/test/fixtures/git_config_with_quotes @@ -0,0 +1,3 @@ +[user] + name = "Cody Veal" + email = "cveal05@gmail.com" diff --git a/test/fixtures/git_config_with_quotes_escapes b/test/fixtures/git_config_with_quotes_escapes new file mode 100644 index 000000000..33332c221 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_escapes @@ -0,0 +1,9 @@ +[custom] + hasnewline = "first\nsecond" + hasbackslash = "foo\\bar" + hasquote = "ab\"cd" + hastrailingbackslash = "word\\" + hasunrecognized = "p\qrs" + hasunescapedquotes = "ab"cd"e" + ordinary = "hello world" + unquoted = good evening diff --git a/test/fixtures/git_config_with_quotes_whitespace_inside b/test/fixtures/git_config_with_quotes_whitespace_inside new file mode 100644 index 000000000..c6014cc61 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_whitespace_inside @@ -0,0 +1,2 @@ +[core] + commentString = "# " diff --git a/test/fixtures/git_config_with_quotes_whitespace_outside b/test/fixtures/git_config_with_quotes_whitespace_outside new file mode 100644 index 000000000..4b1615a51 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_whitespace_outside @@ -0,0 +1,2 @@ +[init] + defaultBranch = "trunk" diff --git a/test/fixtures/index_extended_flags b/test/fixtures/index_extended_flags new file mode 100644 index 000000000..f03713b68 Binary files /dev/null and b/test/fixtures/index_extended_flags differ diff --git a/test/lib/helper.py b/test/lib/helper.py index 1fdc56fd1..58923eaef 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -3,20 +3,46 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = [ + "fixture_path", + "fixture", + "StringProcessAdapter", + "with_rw_directory", + "with_rw_repo", + "with_rw_and_rw_remote_repo", + "PathLikeMock", + "TestBase", + "VirtualEnvironment", + "TestCase", + "SkipTest", + "skipIf", + "GIT_REPO", + "GIT_DAEMON_PORT", + "xfail_if_raises", + "symlinks_supported", + "requires_symlinks", +] + import contextlib +from dataclasses import dataclass from functools import wraps import gc import io import logging import os import os.path as osp +from stat import S_ISLNK, ST_MODE +import subprocess +import sys import tempfile import textwrap import time import unittest import venv +from typing import Union, Type, Tuple import gitdb +import pytest from git.util import rmtree, cwd @@ -29,23 +55,16 @@ GIT_REPO = os.environ.get("GIT_PYTHON_TEST_GIT_REPO_BASE", ospd(ospd(ospd(__file__)))) GIT_DAEMON_PORT = os.environ.get("GIT_PYTHON_TEST_GIT_DAEMON_PORT", "19418") -__all__ = ( - "fixture_path", - "fixture", - "StringProcessAdapter", - "with_rw_directory", - "with_rw_repo", - "with_rw_and_rw_remote_repo", - "TestBase", - "VirtualEnvironment", - "TestCase", - "SkipTest", - "skipIf", - "GIT_REPO", - "GIT_DAEMON_PORT", -) +_logger = logging.getLogger(__name__) + + +@dataclass +class PathLikeMock: + path: str + + def __fspath__(self) -> str: + return self.path -log = logging.getLogger(__name__) # { Routines @@ -74,7 +93,7 @@ def __init__(self, input_string): self.stdout = io.BytesIO(input_string) self.stderr = io.BytesIO() - def wait(self): + def wait(self, stderr=None): return 0 poll = wait @@ -96,8 +115,9 @@ def wrapper(self, *args, **kwargs): try: return func(self, path, *args, **kwargs) except Exception: - log.info( - "Test %s.%s failed, output is at %r\n", + _logger.info( + "%s %s.%s failed, output is at %r\n", + "Test" if func.__name__.startswith("test_") else "Helper", type(self).__name__, func.__name__, path, @@ -146,8 +166,8 @@ def repo_creator(self): os.chdir(rw_repo.working_dir) try: return func(self, rw_repo) - except: # noqa: E722 B001 - log.info("Keeping repo after failure: %s", repo_dir) + except: # noqa: E722 + _logger.info("Keeping repo after failure: %s", repo_dir) repo_dir = None raise finally: @@ -175,7 +195,7 @@ def git_daemon_launched(base_path, ip, port): gd = None try: - if os.name == "nt": + if sys.platform == "win32": # On MINGW-git, daemon exists in Git\mingw64\libexec\git-core\, # but if invoked as 'git daemon', it detaches from parent `git` cmd, # and then CANNOT DIE! @@ -199,7 +219,7 @@ def git_daemon_launched(base_path, ip, port): as_process=True, ) # Yes, I know... fortunately, this is always going to work if sleep time is just large enough. - time.sleep(1.0 if os.name == "nt" else 0.5) + time.sleep(1.0 if sys.platform == "win32" else 0.5) except Exception as ex: msg = textwrap.dedent( """ @@ -212,7 +232,7 @@ def git_daemon_launched(base_path, ip, port): and setting the environment variable GIT_PYTHON_TEST_GIT_DAEMON_PORT to """ ) - log.warning(msg, ex, ip, port, base_path, base_path, exc_info=1) + _logger.warning(msg, ex, ip, port, base_path, base_path, exc_info=1) yield # OK, assume daemon started manually. @@ -221,11 +241,11 @@ def git_daemon_launched(base_path, ip, port): finally: if gd: try: - log.debug("Killing git-daemon...") + _logger.debug("Killing git-daemon...") gd.proc.kill() except Exception as ex: # Either it has died (and we're here), or it won't die, again here... - log.debug("Hidden error while Killing git-daemon: %s", ex, exc_info=1) + _logger.debug("Hidden error while Killing git-daemon: %s", ex, exc_info=1) def with_rw_and_rw_remote_repo(working_tree_ref): @@ -306,8 +326,8 @@ def remote_repo_creator(self): with cwd(rw_repo.working_dir): try: return func(self, rw_repo, rw_daemon_repo) - except: # noqa: E722 B001 - log.info( + except: # noqa: E722 + _logger.info( "Keeping repos after failure: \n rw_repo_dir: %s \n rw_daemon_repo_dir: %s", rw_repo_dir, rw_daemon_repo_dir, @@ -345,24 +365,33 @@ class TestBase(TestCase): """Base class providing default functionality to all tests such as: - Utility functions provided by the TestCase base of the unittest method such as:: + self.fail("todo") self.assertRaises(...) - Class level repository which is considered read-only as it is shared among all test cases in your type. + Access it using:: - self.rorepo # 'ro' stands for read-only + + self.rorepo # 'ro' stands for read-only The rorepo is in fact your current project's git repo. If you refer to specific - shas for your objects, be sure you choose some that are part of the immutable portion - of the project history (so that tests don't fail for others). + shas for your objects, be sure you choose some that are part of the immutable + portion of the project history (so that tests don't fail for others). """ def _small_repo_url(self): """:return: A path to a small, clonable repository""" from git.cmd import Git - return Git.polish_url(osp.join(self.rorepo.working_tree_dir, "git/ext/gitdb/gitdb/ext/smmap")) + return Git.polish_url(self._dependency_repo_dirs["smmap"]) + + def _gitdb_repo_url(self): + """:return: A local gitdb repository reconstructed from merged history""" + from git.cmd import Git + + return Git.polish_url(self._dependency_repo_dirs["gitdb"]) @classmethod def setUpClass(cls): @@ -374,16 +403,31 @@ def setUpClass(cls): gc.collect() cls.rorepo = Repo(GIT_REPO) + cls._dependency_repo_root = tempfile.mkdtemp(prefix="gitpython-dependency-repos-") + cls._dependency_repo_dirs = {} + for name, rev in { + "gitdb": "2da3232f9d58e7761e384ac6d32f7b1ed77a74a2", + "smmap": "8ce61ad5cc4016bffaf25080bc0d69b3acbe8555", + }.items(): + path = osp.join(cls._dependency_repo_root, name) + repo = cls.rorepo.clone(path, shared=True, no_checkout=True) + repo.create_head("master", repo.commit(rev), force=True).checkout() + if name == "smmap": + repo.create_tag("v0.8.1", ref="master~10", message="Test fixture tag", force=True) + repo.git.gc() + repo.close() + cls._dependency_repo_dirs[name] = path @classmethod def tearDownClass(cls): cls.rorepo.git.clear_cache() cls.rorepo.git = None + rmtree(cls._dependency_repo_root) def _make_file(self, rela_path, data, repo=None): """ - Create a file at the given path relative to our repository, filled - with the given data. + Create a file at the given path relative to our repository, filled with the + given data. :return: An absolute path to the created file. """ @@ -400,13 +444,26 @@ class VirtualEnvironment: __slots__ = ("_env_dir",) def __init__(self, env_dir, *, with_pip): - if os.name == "nt": + if sys.platform == "win32": self._env_dir = osp.realpath(env_dir) venv.create(self.env_dir, symlinks=False, with_pip=with_pip) else: self._env_dir = env_dir venv.create(self.env_dir, symlinks=True, with_pip=with_pip) + if with_pip: + # The upgrade_deps parameter to venv.create is 3.9+ only, so do it this way. + command = [ + self.python, + "-m", + "pip", + "install", + "--upgrade", + "pip", + 'setuptools; python_version<"3.12"', + ] + subprocess.check_output(command) + @property def env_dir(self): """The top-level directory of the environment.""" @@ -428,10 +485,56 @@ def sources(self): return os.path.join(self.env_dir, "src") def _executable(self, basename): - if os.name == "nt": + if sys.platform == "win32": path = osp.join(self.env_dir, "Scripts", basename + ".exe") else: path = osp.join(self.env_dir, "bin", basename) if osp.isfile(path) or osp.islink(path): return path raise RuntimeError(f"no regular file or symlink {path!r}") + + +def symlinks_supported() -> bool: + """Check whether this process can actually create a symlink. + + On Windows the platform alone doesn't decide it: creating a symlink needs either + Developer Mode or SeCreateSymbolicLinkPrivilege, and an unprivileged process gets + OSError (WinError 1314) instead. + """ + with tempfile.TemporaryDirectory(prefix="gitpython-symlink-check-") as temp_dir: + link_path = osp.join(temp_dir, "link") + try: + os.symlink("missing-target", link_path) + except (NotImplementedError, OSError): + return False + return S_ISLNK(os.lstat(link_path)[ST_MODE]) + + +requires_symlinks = pytest.mark.skipif( + not symlinks_supported(), + reason="symlinks are unavailable, or need privileges this process doesn't have", +) + + +@contextlib.contextmanager +def xfail_if_raises( + condition: bool, + *, + raises: Union[Type[BaseException], Tuple[Type[BaseException], ...]], + reason: str = "", + strict: bool = False, +): + """Approximates the behavior of @pytest.mark.xfail(..., raises=...) as a context + manager that can be used within a test, such as when the condition is complex or has + side effects + + One difference is it will not report XPASS if the test passes, but setting `strict` + simulates it by raising an exception""" + try: + yield + except raises: + if condition: + pytest.xfail(reason) + raise + if strict and condition: + pytest.fail("[XPASS(strict)] " + reason) diff --git a/test/performance/__init__.py b/test/performance/__init__.py index e69de29bb..56b5d89db 100644 --- a/test/performance/__init__.py +++ b/test/performance/__init__.py @@ -0,0 +1,2 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/test/performance/lib.py b/test/performance/lib.py index d08e1027f..c24599986 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -1,17 +1,18 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Support library for tests.""" +"""Support library for performance tests.""" import logging import os +import os.path as osp import tempfile from git import Repo from git.db import GitCmdObjectDB, GitDB -from test.lib import TestBase from git.util import rmtree -import os.path as osp + +from test.lib import TestBase # { Invariants diff --git a/test/performance/test_commit.py b/test/performance/test_commit.py index 00d768f0a..b943f1975 100644 --- a/test/performance/test_commit.py +++ b/test/performance/test_commit.py @@ -10,9 +10,11 @@ from time import time import sys -from .lib import TestBigRepoRW -from git import Commit from gitdb import IStream + +from git import Commit + +from test.performance.lib import TestBigRepoRW from test.test_commit import TestCommitSerialization diff --git a/test/performance/test_odb.py b/test/performance/test_odb.py index 00e245fb7..fdbbeb8c3 100644 --- a/test/performance/test_odb.py +++ b/test/performance/test_odb.py @@ -6,7 +6,7 @@ import sys from time import time -from .lib import TestBigRepoR +from test.performance.lib import TestBigRepoR class TestObjDBPerformance(TestBigRepoR): diff --git a/test/performance/test_streams.py b/test/performance/test_streams.py index ba5cbe415..f6ffeba8e 100644 --- a/test/performance/test_streams.py +++ b/test/performance/test_streams.py @@ -5,18 +5,18 @@ import gc import os +import os.path as osp import subprocess import sys from time import time -from test.lib import with_rw_repo -from git.util import bin_to_hex from gitdb import LooseObjectDB, IStream from gitdb.test.lib import make_memory_file -import os.path as osp +from git.util import bin_to_hex -from .lib import TestBigRepoR +from test.lib import with_rw_repo +from test.performance.lib import TestBigRepoR class TestObjDBPerformance(TestBigRepoR): @@ -25,8 +25,8 @@ class TestObjDBPerformance(TestBigRepoR): @with_rw_repo("HEAD", bare=True) def test_large_data_streaming(self, rwrepo): - # TODO: This part overlaps with the same file in gitdb.test.performance.test_stream. - # It should be shared if possible. + # TODO: This part overlaps with the same file in + # gitdb.test.performance.test_stream. It should be shared if possible. ldb = LooseObjectDB(osp.join(rwrepo.git_dir, "objects")) for randomize in range(2): diff --git a/test/test_actor.py b/test/test_actor.py index caf095739..baf6545f1 100644 --- a/test/test_actor.py +++ b/test/test_actor.py @@ -3,9 +3,10 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from test.lib import TestBase from git import Actor +from test.lib import TestBase + class TestActor(TestBase): def test_from_string_should_separate_name_and_email(self): @@ -26,6 +27,26 @@ def test_from_string_should_handle_just_name(self): self.assertEqual("Michael Trier", a.name) self.assertEqual(None, a.email) + def test_from_string_handles_unterminated_email_without_regex_backtracking(self): + value = "A" * 20_000 + " \n y "), Actor("x", "a")) + + def test_from_string_uses_git_delimiters(self): + for value, expected in ( + ("Name ", Actor("Name", "e>", Actor("Name", "email")), + ("Name", Actor("Name", "email")), + (" <>", Actor("", "")), + ("Name ", Actor("Name email>", None)), + ): + self.assertEqual(Actor._from_string(value), expected) + def test_should_display_representation(self): a = Actor._from_string("Michael Trier ") self.assertEqual('">', repr(a)) diff --git a/test/test_autointerrupt.py b/test/test_autointerrupt.py new file mode 100644 index 000000000..645ec402c --- /dev/null +++ b/test/test_autointerrupt.py @@ -0,0 +1,33 @@ +from git.cmd import Git + + +class _DummyProc: + """Minimal stand-in for subprocess.Popen used to exercise AutoInterrupt. + + We deliberately raise AttributeError from terminate() to simulate interpreter + shutdown on Windows where subprocess internals (e.g. subprocess._winapi) may + already be torn down. + """ + + stdin = None + stdout = None + stderr = None + + def poll(self): + return None + + def terminate(self): + raise AttributeError("TerminateProcess") + + def wait(self): # pragma: no cover - should not be reached in this test + raise AssertionError("wait() should not be called if terminate() fails") + + +def test_autointerrupt_terminate_ignores_attributeerror(): + ai = Git.AutoInterrupt(_DummyProc(), args=["git", "rev-list"]) + + # Should not raise, even if terminate() triggers AttributeError. + ai._terminate() + + # Ensure the reference is cleared to avoid repeated attempts. + assert ai.proc is None diff --git a/test/test_base.py b/test/test_base.py index 74f342071..86bcc5c79 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -5,18 +5,18 @@ import gc import os +import os.path as osp import sys import tempfile from unittest import skipIf from git import Repo -from git.objects import Blob, Tree, Commit, TagObject +from git.objects import Blob, Commit, TagObject, Tree +import git.objects.base as base from git.objects.util import get_object_type_by_name -from test.lib import TestBase as _TestBase, with_rw_repo, with_rw_and_rw_remote_repo -from git.util import hex_to_bin, HIDE_WINDOWS_FREEZE_ERRORS +from git.util import HIDE_WINDOWS_FREEZE_ERRORS, hex_to_bin -import git.objects.base as base -import os.path as osp +from test.lib import TestBase as _TestBase, with_rw_and_rw_remote_repo, with_rw_repo class TestBase(_TestBase): @@ -72,7 +72,10 @@ def test_base_object(self): self.assertEqual(item, item.stream_data(tmpfile)) tmpfile.seek(0) self.assertEqual(tmpfile.read(), data) - os.remove(tmpfile.name) # Do it this way so we can inspect the file on failure. + + # Remove the file this way, instead of with a context manager or "finally", + # so it is only removed on success, and we can inspect the file on failure. + os.remove(tmpfile.name) # END for each object type to create # Each has a unique sha. @@ -127,13 +130,13 @@ def test_add_unicode(self, rw_repo): with open(file_path, "wb") as fp: fp.write(b"something") - if os.name == "nt": + if sys.platform == "win32": # On Windows, there is no way this works, see images on: # https://github.com/gitpython-developers/GitPython/issues/147#issuecomment-68881897 # Therefore, it must be added using the Python implementation. rw_repo.index.add([file_path]) - # However, when the test winds down, rmtree fails to delete this file, which is recognized - # as ??? only. + # However, when the test winds down, rmtree fails to delete this file, which + # is recognized as ??? only. else: # On POSIX, we can just add Unicode files without problems. rw_repo.git.add(rw_repo.working_dir) diff --git a/test/test_blob.py b/test/test_blob.py index ff59c67ea..affaa60fc 100644 --- a/test/test_blob.py +++ b/test/test_blob.py @@ -3,9 +3,10 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from test.lib import TestBase from git import Blob +from test.lib import TestBase + class TestBlob(TestBase): def test_mime_type_should_return_mime_type_for_known_types(self): diff --git a/test/test_clone.py b/test/test_clone.py index dcab7ad6f..a6c3db6f9 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -1,15 +1,24 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import os +import os.path as osp +import pathlib +import sys +import tempfile +from unittest import skip +from unittest import mock + +from git import Git, GitCommandError, Repo +from git.exc import UnsafeOptionError, UnsafeProtocolError + +from test.lib import TestBase, with_rw_directory, with_rw_repo, PathLikeMock + from pathlib import Path import re import git - -from .lib import ( - TestBase, - with_rw_directory, -) +import pytest class TestClone(TestBase): @@ -19,8 +28,8 @@ def test_checkout_in_non_empty_dir(self, rw_dir): garbage_file = non_empty_dir / "not-empty" garbage_file.write_text("Garbage!") - # Verify that cloning into the non-empty dir fails while complaining about - # the target directory not being empty/non-existent. + # Verify that cloning into the non-empty dir fails while complaining about the + # target directory not being empty/non-existent. try: self.rorepo.clone(non_empty_dir) except git.GitCommandError as exc: @@ -32,3 +41,397 @@ def test_checkout_in_non_empty_dir(self, rw_dir): ) else: self.fail("GitCommandError not raised") + + @with_rw_directory + def test_clone_from_pathlib(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + + Repo.clone_from(pathlib.Path(original_repo.git_dir), pathlib.Path(rw_dir) / "clone_pathlib") + + @with_rw_directory + def test_clone_from_pathlike(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + Repo.clone_from(PathLikeMock(original_repo.git_dir), PathLikeMock(os.path.join(rw_dir, "clone_pathlike"))) + + @with_rw_directory + def test_clone_from_pathlib_withConfig(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + + cloned = Repo.clone_from( + original_repo.git_dir, + pathlib.Path(rw_dir) / "clone_pathlib_withConfig", + multi_options=[ + "--recurse-submodules=repo", + "--config core.filemode=false", + "--config submodule.repo.update=checkout", + "--config filter.lfs.clean='git-lfs clean -- %f'", + ], + allow_unsafe_options=True, + ) + + self.assertEqual(cloned.config_reader().get_value("submodule", "active"), "repo") + self.assertEqual(cloned.config_reader().get_value("core", "filemode"), False) + self.assertEqual(cloned.config_reader().get_value('submodule "repo"', "update"), "checkout") + self.assertEqual( + cloned.config_reader().get_value('filter "lfs"', "clean"), + "git-lfs clean -- %f", + ) + + def test_clone_from_with_path_contains_unicode(self): + with tempfile.TemporaryDirectory() as tmpdir: + unicode_dir_name = "\u0394" + path_with_unicode = os.path.join(tmpdir, unicode_dir_name) + os.makedirs(path_with_unicode) + + try: + Repo.clone_from( + url=self._small_repo_url(), + to_path=path_with_unicode, + # Local clones hardlink the reconstructed smmap repository's + # read-only metadata, which Windows cannot remove at cleanup. + no_hardlinks=True, + ) + except UnicodeEncodeError: + self.fail("Raised UnicodeEncodeError") + + @with_rw_directory + @skip( + """The referenced repository was removed, and one needs to set up a new + password controlled repo under the org's control.""" + ) + def test_leaking_password_in_clone_logs(self, rw_dir): + password = "fakepassword1234" + try: + Repo.clone_from( + url="https://fakeuser:{}@fakerepo.example.com/testrepo".format(password), + to_path=rw_dir, + ) + except GitCommandError as err: + assert password not in str(err), "The error message '%s' should not contain the password" % err + # Working example from a blank private project. + Repo.clone_from( + url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", + to_path=rw_dir, + ) + + @with_rw_repo("HEAD") + def test_clone_unsafe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"--upl='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + f"-utouch {tmp_file}; false", + f"-futouch${{IFS}}{tmp_file}; false", + f"-qutouch${{IFS}}{tmp_file}; false", + "--config=protocol.ext.allow=always", + "--conf=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + "-cprotocol.ext.allow=always", + "-vcprotocol.ext.allow=always", + f"--template={tmp_dir}", + f"--bundle-uri=file://{tmp_dir}", + f"--separate-git-dir={tmp_dir / 'git-dir'}", + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) + assert not tmp_file.exists() + + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, + {"upl": f"touch {tmp_file}"}, + {"u": f"touch {tmp_file}"}, + {"config": "protocol.ext.allow=always"}, + {"conf": "protocol.ext.allow=always"}, + {"c": "protocol.ext.allow=always"}, + {"template": tmp_dir}, + {"bundle_uri": f"file://{tmp_dir}"}, + {"separate_git_dir": tmp_dir / "git-dir"}, + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, **unsafe_option) + assert not tmp_file.exists() + + @with_rw_repo("HEAD") + def test_clone_unsafe_options_abbreviated(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upl='touch {tmp_file}'", + f"--upload-pac='touch {tmp_file}'", + "--conf=protocol.ext.allow=always", + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) + assert not tmp_file.exists() + + unsafe_kwargs = [ + {"upl": f"touch {tmp_file}"}, + {"upload_pac": f"touch {tmp_file}"}, + {"conf": "protocol.ext.allow=always"}, + ] + for unsafe_option in unsafe_kwargs: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, **unsafe_option) + assert not tmp_file.exists() + + @with_rw_repo("HEAD") + def test_clone_unsafe_options_are_checked_after_splitting_multi_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + payload = "--single-branch --config protocol.ext.allow=always" + + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, multi_options=[payload]) + + @pytest.mark.xfail( + sys.platform == "win32", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_clone_unsafe_options must be adjusted in the " + "same way. Until then, test_clone_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) + @with_rw_repo("HEAD") + def test_clone_unsafe_options_allowed(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not tmp_file.exists() + # The options will be allowed, but the command will fail. + with self.assertRaises(GitCommandError): + rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) + assert tmp_file.exists() + tmp_file.unlink() + + unsafe_options = [ + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not destination.exists() + rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_safe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + options = [ + "--depth=1", + "--single-branch", + "--origin upload", + "-q", + "-oupstream", + ] + for option in options: + destination = tmp_dir / option + assert not destination.exists() + rw_repo.clone(destination, multi_options=[option]) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_from_unsafe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + f"-utouch {tmp_file}; false", + f"-futouch${{IFS}}{tmp_file}; false", + f"-qutouch${{IFS}}{tmp_file}; false", + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + "-cprotocol.ext.allow=always", + "-vcprotocol.ext.allow=always", + f"--separate-git-dir={tmp_dir / 'git-dir'}", + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + Repo.clone_from(rw_repo.working_dir, tmp_dir, multi_options=[unsafe_option]) + assert not tmp_file.exists() + + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, + {"u": f"touch {tmp_file}"}, + {"config": "protocol.ext.allow=always"}, + {"c": "protocol.ext.allow=always"}, + {"separate_git_dir": tmp_dir / "git-dir"}, + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + Repo.clone_from(rw_repo.working_dir, tmp_dir, **unsafe_option) + assert not tmp_file.exists() + + @with_rw_repo("HEAD") + def test_clone_from_unsafe_options_are_checked_after_splitting_multi_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + payload = "--single-branch --config protocol.ext.allow=always" + + with self.assertRaises(UnsafeOptionError): + Repo.clone_from(rw_repo.working_dir, tmp_dir, multi_options=[payload]) + + @pytest.mark.xfail( + sys.platform == "win32", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_clone_from_unsafe_options must be adjusted in the " + "same way. Until then, test_clone_from_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) + @with_rw_repo("HEAD") + def test_clone_from_unsafe_options_allowed(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not tmp_file.exists() + # The options will be allowed, but the command will fail. + with self.assertRaises(GitCommandError): + Repo.clone_from( + rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True + ) + assert tmp_file.exists() + tmp_file.unlink() + + unsafe_options = [ + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not destination.exists() + Repo.clone_from( + rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True + ) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_from_safe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + options = [ + "--depth=1", + "--single-branch", + "-q", + ] + for option in options: + destination = tmp_dir / option + assert not destination.exists() + Repo.clone_from(rw_repo.common_dir, destination, multi_options=[option]) + assert destination.exists() + + def test_clone_from_unsafe_protocol(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + "fd::17/foo", + ] + for url in urls: + with self.assertRaises(UnsafeProtocolError): + Repo.clone_from(url, tmp_dir / "repo") + assert not tmp_file.exists() + + def test_clone_from_does_not_expand_environment_variables_in_url(self): + urls = [ + "https://example.com/$GITPYTHON_TEST_SECRET/repo.git", + "https://example.com/${GITPYTHON_TEST_SECRET}/repo.git", + "https://example.com/%GITPYTHON_TEST_SECRET%/repo.git", + ] + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + for url in urls: + with mock.patch.object(Git, "_call_process", side_effect=RuntimeError) as call_process: + with self.assertRaises(RuntimeError): + Repo.clone_from(url, "unused") + + assert call_process.call_args[0][3] == url + + @with_rw_directory + def test_clone_from_does_not_expand_environment_variables_in_stored_url(self, rw_dir): + url = pathlib.Path(rw_dir) / "$GITPYTHON_TEST_SECRET" / "source" + Git().init(url) + + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + cloned = Repo.clone_from(url, pathlib.Path(rw_dir) / "clone") + + assert cloned.remotes.origin.url == Git.polish_url(str(url), expand_vars=False) + + def test_clone_from_checks_polished_url_for_unsafe_protocol(self): + with mock.patch.object(Git, "polish_url", return_value="ext::command"): + with mock.patch.object(Git, "_call_process") as call_process: + with self.assertRaises(UnsafeProtocolError): + Repo.clone_from("$GITPYTHON_TEST_URL", "unused") + + call_process.assert_not_called() + + def test_polish_url_does_not_expand_environment_variables_for_cygwin(self): + urls = ["$GITPYTHON_TEST_SECRET/repo", "user@example.com:$GITPYTHON_TEST_SECRET/repo"] + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + for url in urls: + assert Git.polish_url(url, is_cygwin=True, expand_vars=False) == url + + def test_clone_from_unsafe_protocol_allowed(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + "fd::/foo", + ] + for url in urls: + # The URL will be allowed into the command, but the command will + # fail since we don't have that protocol enabled in the Git config file. + with self.assertRaises(GitCommandError): + Repo.clone_from(url, tmp_dir / "repo", allow_unsafe_protocols=True) + assert not tmp_file.exists() + + def test_clone_from_unsafe_protocol_allowed_and_enabled(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + ] + allow_ext = [ + "--config=protocol.ext.allow=always", + ] + for url in urls: + # The URL will be allowed into the command, and the protocol is enabled, + # but the command will fail since it can't read from the remote repo. + assert not tmp_file.exists() + with self.assertRaises(GitCommandError): + Repo.clone_from( + url, + tmp_dir / "repo", + multi_options=allow_ext, + allow_unsafe_protocols=True, + allow_unsafe_options=True, + ) + assert tmp_file.exists() + tmp_file.unlink() diff --git a/test/test_commit.py b/test/test_commit.py index fddb91c17..431269b29 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -6,29 +6,33 @@ import copy from datetime import datetime from io import BytesIO +import tempfile +import os.path as osp import re import sys import time from unittest.mock import Mock -from git import ( - Commit, - Actor, -) -from git import Repo +from gitdb import IStream + +from git import Actor, Commit, Repo +from git.exc import UnsafeOptionError from git.objects.util import tzoffset, utc from git.repo.fun import touch -from test.lib import TestBase, with_rw_repo, fixture_path, StringProcessAdapter -from test.lib import with_rw_directory -from gitdb import IStream -import os.path as osp +from test.lib import ( + StringProcessAdapter, + TestBase, + fixture_path, + with_rw_directory, + with_rw_repo, +) class TestCommitSerialization(TestBase): def assert_commit_serialization(self, rwrepo, commit_id, print_performance_info=False): - """Traverse all commits in the history of commit identified by commit_id and check - if the serialization works. + """Traverse all commits in the history of commit identified by commit_id and + check if the serialization works. :param print_performance_info: If True, we will show how fast we are. """ @@ -133,9 +137,12 @@ def test_stats(self): commit = self.rorepo.commit("33ebe7acec14b25c5f84f35a664803fcab2f7781") stats = commit.stats - def check_entries(d): + def check_entries(d, has_change_type=False): assert isinstance(d, dict) - for key in ("insertions", "deletions", "lines"): + keys = ("insertions", "deletions", "lines") + if has_change_type: + keys += ("change_type",) + for key in keys: assert key in d # END assertion helper @@ -146,7 +153,7 @@ def check_entries(d): assert "files" in stats.total for _filepath, d in stats.files.items(): - check_entries(d) + check_entries(d, True) # END for each stated file # Check that data is parsed properly. @@ -269,20 +276,36 @@ def test_iteration(self): assert ltd_commits and len(ltd_commits) < len(all_commits) # Show commits of multiple paths, resulting in a union of commits. - less_ltd_commits = list(Commit.iter_items(self.rorepo, "master", paths=("CHANGES", "AUTHORS"))) + less_ltd_commits = list(Commit.iter_items(self.rorepo, "HEAD", paths=("CHANGES", "AUTHORS"))) assert len(ltd_commits) < len(less_ltd_commits) class Child(Commit): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - child_commits = list(Child.iter_items(self.rorepo, "master", paths=("CHANGES", "AUTHORS"))) + child_commits = list(Child.iter_items(self.rorepo, "HEAD", paths=("CHANGES", "AUTHORS"))) assert type(child_commits[0]) is Child def test_iter_items(self): # pretty not allowed. self.assertRaises(ValueError, Commit.iter_items, self.rorepo, "master", pretty="raw") + def test_iter_items_rejects_unsafe_revision(self): + with tempfile.TemporaryDirectory() as tdir: + marker = osp.join(tdir, "pwn") + self.assertRaises(UnsafeOptionError, Commit.iter_items, self.rorepo, f"--output={marker}") + + def test_iter_items_rejects_unsafe_options(self): + with tempfile.TemporaryDirectory() as tdir: + marker = osp.join(tdir, "pwn") + with self.assertRaises(UnsafeOptionError): + list(Commit.iter_items(self.rorepo, "HEAD", output=marker)) + + def test_count_rejects_unsafe_options(self): + with tempfile.NamedTemporaryFile() as marker: + with self.assertRaises(UnsafeOptionError): + self.rorepo.head.commit.count(output=marker.name) + def test_rev_list_bisect_all(self): """ 'git rev-list --bisect-all' returns additional information @@ -304,6 +327,11 @@ def test_rev_list_bisect_all(self): for sha1, commit in zip(expected_ids, commits): self.assertEqual(sha1, commit.hexsha) + def test_iter_from_invalid_process_or_stream(self): + for source, error in ((Mock(wait=Mock(), stdout=None), ValueError), (object(), TypeError)): + with self.assertRaises(error): + list(Commit._iter_from_process_or_stream(self.rorepo, source)) + @with_rw_directory def test_ambiguous_arg_iteration(self, rw_dir): rw_repo = Repo.init(osp.join(rw_dir, "test_ambiguous_arg")) @@ -317,8 +345,9 @@ def test_count(self): self.assertEqual(self.rorepo.tag("refs/tags/0.1.5").commit.count(), 143) def test_list(self): - # This doesn't work anymore, as we will either attempt getattr with bytes, or compare 20 byte string - # with actual 20 byte bytes. This usage makes no sense anyway. + # This doesn't work anymore, as we will either attempt getattr with bytes, or + # compare 20 byte string with actual 20 byte bytes. This usage makes no sense + # anyway. assert isinstance( Commit.list_items(self.rorepo, "0.1.5", max_count=5)["5117c9c8a4d3af19a9958677e45cda9269de1541"], Commit, @@ -383,8 +412,8 @@ def test_serialization_unicode_support(self): self.assertEqual(cmt.author.name, ncmt.author.name) self.assertEqual(cmt.message, ncmt.message) - # Actually, it can't be printed in a shell as repr wants to have ascii only - # it appears. + # Actually, it can't be printed in a shell as repr wants to have ascii only it + # appears. cmt.author.__repr__() def test_invalid_commit(self): @@ -498,7 +527,8 @@ def test_trailers(self): KEY_2 = "Key" VALUE_2 = "Value with inner spaces" - # Check that the following trailer example is extracted from multiple msg variations. + # Check that the following trailer example is extracted from multiple msg + # variations. TRAILER = f"{KEY_1}: {VALUE_1_1}\n{KEY_2}: {VALUE_2}\n{KEY_1}: {VALUE_1_2}" msgs = [ f"Subject\n\n{TRAILER}\n", @@ -511,7 +541,7 @@ def test_trailers(self): ), ] for msg in msgs: - commit = copy.copy(self.rorepo.commit("master")) + commit = copy.copy(self.rorepo.commit("HEAD")) commit.message = msg assert commit.trailers_list == [ (KEY_1, VALUE_1_1), @@ -534,13 +564,13 @@ def test_trailers(self): ] for msg in msgs: - commit = copy.copy(self.rorepo.commit("master")) + commit = copy.copy(self.rorepo.commit("HEAD")) commit.message = msg assert commit.trailers_list == [] assert commit.trailers_dict == {} # Check that only the last key value paragraph is evaluated. - commit = copy.copy(self.rorepo.commit("master")) + commit = copy.copy(self.rorepo.commit("HEAD")) commit.message = f"Subject\n\nMultiline\nBody\n\n{KEY_1}: {VALUE_1_1}\n\n{KEY_2}: {VALUE_2}\n" assert commit.trailers_list == [(KEY_2, VALUE_2)] assert commit.trailers_dict == {KEY_2: [VALUE_2]} @@ -559,3 +589,142 @@ def test_commit_co_authors(self): Actor("test_user_2", "another_user-email@github.com"), Actor("test_user_3", "test_user_3@github.com"), ] + + @with_rw_directory + def test_create_from_tree_with_trailers_dict(self, rw_dir): + """Test that create_from_tree supports adding trailers via a dict.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_trailers_dict")) + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + + trailers = {"Issue": "123", "Signed-off-by": "Test User "} + commit = Commit.create_from_tree( + rw_repo, + tree, + "Test commit with trailers", + head=True, + trailers=trailers, + ) + + assert "Issue: 123" in commit.message + assert "Signed-off-by: Test User " in commit.message + assert commit.trailers_dict == { + "Issue": ["123"], + "Signed-off-by": ["Test User "], + } + + @with_rw_directory + def test_create_from_tree_with_trailers_list(self, rw_dir): + """Test that create_from_tree supports adding trailers via a list of tuples.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_trailers_list")) + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + + trailers = [ + ("Signed-off-by", "Alice "), + ("Signed-off-by", "Bob "), + ("Issue", "456"), + ] + commit = Commit.create_from_tree( + rw_repo, + tree, + "Test commit with multiple trailers", + head=True, + trailers=trailers, + ) + + assert "Signed-off-by: Alice " in commit.message + assert "Signed-off-by: Bob " in commit.message + assert "Issue: 456" in commit.message + assert commit.trailers_dict == { + "Signed-off-by": ["Alice ", "Bob "], + "Issue": ["456"], + } + + @with_rw_directory + def test_create_from_tree_with_non_utf8_trailers(self, rw_dir): + """Test that trailer creation and parsing respect the configured commit encoding.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_trailers_non_utf8")) + with rw_repo.config_writer() as writer: + writer.set_value("i18n", "commitencoding", "ISO-8859-1") + + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + + commit = Commit.create_from_tree( + rw_repo, + tree, + "RÊsumÊ", + head=True, + trailers={"Reviewed-by": "AndrÊ "}, + ) + + assert commit.encoding == "ISO-8859-1" + assert "RÊsumÊ" in commit.message + assert "Reviewed-by: AndrÊ " in commit.message + assert commit.trailers_list == [("Reviewed-by", "AndrÊ ")] + + @with_rw_directory + def test_trailers_list_with_non_utf8_message_bytes(self, rw_dir): + """Test that trailer parsing handles non-UTF-8 commit message bytes.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_trailers_non_utf8_bytes")) + with rw_repo.config_writer() as writer: + writer.set_value("i18n", "commitencoding", "ISO-8859-1") + + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + + commit = Commit.create_from_tree( + rw_repo, + tree, + "RÊsumÊ", + head=True, + trailers={"Reviewed-by": "AndrÊ "}, + ) + + bytes_commit = Commit( + rw_repo, + commit.binsha, + message=commit.message.encode(commit.encoding), + encoding=commit.encoding, + ) + + assert bytes_commit.trailers_list == [("Reviewed-by", "AndrÊ ")] + + def test_interpret_trailers_encodes_before_launching_process(self): + """Test that encoding failures happen before spawning interpret-trailers.""" + repo = Mock() + repo.git = Mock() + repo.git.GIT_PYTHON_GIT_EXECUTABLE = "git" + + with self.assertRaises(UnicodeEncodeError): + Commit._interpret_trailers(repo, "Euro: â‚Ŧ", ["--parse"], encoding="ISO-8859-1") + + repo.git.execute.assert_not_called() + + @with_rw_directory + def test_index_commit_with_trailers(self, rw_dir): + """Test that IndexFile.commit() supports adding trailers.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_index_trailers")) + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + + trailers = {"Reviewed-by": "Reviewer "} + commit = rw_repo.index.commit( + "Test index commit with trailers", + trailers=trailers, + ) + + assert "Reviewed-by: Reviewer " in commit.message + assert commit.trailers_dict == { + "Reviewed-by": ["Reviewer "], + } diff --git a/test/test_config.py b/test/test_config.py index f493c5672..28bb12043 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -7,16 +7,18 @@ import io import os import os.path as osp +import subprocess +import sys from unittest import mock import pytest from git import GitConfigParser +from git.compat import defenc from git.config import _OMD, cp -from git.util import rmfile +from git.util import cwd, rmfile from test.lib import SkipTest, TestCase, fixture_path, with_rw_directory - _tc_lock_fpaths = osp.join(osp.dirname(__file__), "fixtures/*.lock") @@ -41,7 +43,7 @@ def _to_memcache(self, file_path): return sio def test_read_write(self): - # writer must create the exact same file as the one read before + # The writer must create the exact same file as the one read before. for filename in ("git_config", "git_config_global"): file_obj = self._to_memcache(fixture_path(filename)) with GitConfigParser(file_obj, read_only=False) as w_config: @@ -56,7 +58,8 @@ def test_read_write(self): self._to_memcache(fixture_path(filename)).getvalue(), ) - # Creating an additional config writer must fail due to exclusive access. + # Creating an additional config writer must fail due to exclusive + # access. with self.assertRaises(IOError): GitConfigParser(file_obj, read_only=False) @@ -91,8 +94,8 @@ def test_includes_order(self): r_config.read() # Enforce reading. # Simple inclusions, again checking them taking precedence. assert r_config.get_value("sec", "var0") == "value0_included" - # This one should take the git_config_global value since included - # values must be considered as soon as they get them. + # This one should take the git_config_global value since included values + # must be considered as soon as they get them. assert r_config.get_value("diff", "tool") == "meld" try: # FIXME: Split this assertion out somehow and mark it xfail (or fix it). @@ -109,7 +112,8 @@ def test_lock_reentry(self, rw_dir): # Entering again locks the file again... with gcp as cw: cw.set_value("include", "some_other_value", "b") - # ...so creating an additional config writer must fail due to exclusive access. + # ...so creating an additional config writer must fail due to exclusive + # access. with self.assertRaises(IOError): GitConfigParser(fpl, read_only=False) # but work when the lock is removed @@ -139,6 +143,272 @@ def test_multi_line_config(self): ) self.assertEqual(len(config.sections()), 23) + def test_config_value_with_trailing_new_line(self): + config_content = b'[section-header]\nkey:"value\n"' + config_file = io.BytesIO(config_content) + config_file.name = "multiline_value.config" + + git_config = GitConfigParser(config_file) + git_config.read() # This should not throw an exception + + @with_rw_directory + def test_rewriting_multiline_value_does_not_create_option(self, rw_dir): + config_path = osp.join(rw_dir, "config") + with open(config_path, "wb") as config_file: + config_file.write(b'[core]\n\tzzz = "A\\nhooksPath = ../evil-hooks\\\n"\n') + + with GitConfigParser(config_path, read_only=False) as git_config: + self.assertEqual(git_config.get_value("core", "zzz"), "A\nhooksPath = ../evil-hooks") + git_config.set_value("user", "name", "Test User") + + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertEqual(git_config.get_value("core", "zzz"), "A\nhooksPath = ../evil-hooks") + self.assertFalse(git_config.has_option("core", "hooksPath")) + self.assertEqual( + subprocess.run(["git", "config", "--file", config_path, "--get", "core.hooksPath"]).returncode, 1 + ) + + @with_rw_directory + def test_writer_escapes_special_characters_without_newline(self, rw_dir): + config_path = osp.join(rw_dir, "config") + values = { + "tab": "\tvalue\t", + "backspace": "a\bb", + "quote": 'a"b', + "backslash": "a\\qb", + "hash": "value#fragment", + "semicolon": "value;fragment", + "leading": " value", + "trailing": "value ", + } + + with GitConfigParser(config_path, read_only=False) as git_config: + for key, value in values.items(): + git_config.set_value("section", key, value) + + with GitConfigParser(config_path, read_only=True) as git_config: + for key, value in values.items(): + self.assertEqual(git_config.get_value("section", key), value) + self.assertEqual( + subprocess.run( + ["git", "config", "--file", config_path, "--get", "section.%s" % key], + stdout=subprocess.PIPE, + check=True, + ).stdout, + value.encode(defenc) + b"\n", + ) + with open(config_path, "rb") as config_file: + self.assertNotIn(b"\x08", config_file.read()) + + @with_rw_directory + def test_writer_preserves_escaped_and_non_ascii_values_safely(self, rw_dir): + config_path = osp.join(rw_dir, "config") + with open(config_path, "wb") as config_file: + config_file.write( + ( + '[section]\nnewline = "first\\nsecond"\nquote = "a\\"b"\nbackslash = "a\\\\b"\n' + 'unicode = "cafÊ\\\\path"\n' + ).encode(defenc) + ) + + with GitConfigParser(config_path, read_only=False) as config: + config.set_value("unrelated", "key", "value") + + expected = { + "newline": "first\nsecond", + "quote": 'a"b', + "backslash": "a\\b", + "unicode": "cafÊ\\path", + } + with GitConfigParser(config_path, read_only=True) as config: + for key, value in expected.items(): + self.assertEqual( + config.get_value("section", key), + value, + "GitPython should preserve values when rewriting unrelated entries", + ) + self.assertEqual( + subprocess.run( + ["git", "config", "--file", config_path, "--get", "section.%s" % key], + stdout=subprocess.PIPE, + check=True, + ).stdout, + value.encode(defenc) + b"\n", + "git should read rewritten values with the same semantics", + ) + + with open(config_path, "rb") as config_file: + contents = config_file.read() + self.assertNotIn(b"\r", contents, "the writer should never emit carriage returns") + self.assertNotIn(b"\x00", contents, "the writer should never emit NUL bytes") + + for name, value in (("return", b"first\\rsecond"), ("nul", b"first\x00second")): + unsafe_path = osp.join(rw_dir, "%s-config" % name) + unsafe_contents = b'[section]\nvalue = "' + value + b'"\n' + with open(unsafe_path, "wb") as config_file: + config_file.write(unsafe_contents) + with self.assertRaisesRegex( + ValueError, + "CR or NUL", + msg="unsafe existing values should abort rewrites", + ): + with GitConfigParser(unsafe_path, read_only=False) as config: + config.set_value("unrelated", "key", "value") + with open(unsafe_path, "rb") as config_file: + self.assertEqual( + config_file.read(), + unsafe_contents, + "rejected rewrites should leave the original file unchanged", + ) + + @with_rw_directory + def test_set_value_rejects_config_injection(self, rw_dir): + config_path = osp.join(rw_dir, "config") + payload = "foo\n[core]\nhooksPath=/tmp/hooks" + + with GitConfigParser(config_path, read_only=False) as git_config: + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set_value("user", "name", payload) + + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertFalse(git_config.has_section("user")) + self.assertFalse(git_config.has_section("core")) + + @with_rw_directory + def test_set_value_rejects_unsafe_section_and_option_names(self, rw_dir): + config_path = osp.join(rw_dir, "config") + bad_keys = ("user]\n[core", "user]\r[core", "user]\x00[core") + + with GitConfigParser(config_path, read_only=False) as git_config: + git_config.add_section("user") + for bad_key in bad_keys: + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.add_section(bad_key) + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set(bad_key, "hooksPath", "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set("user", bad_key, "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set_value(bad_key, "hooksPath", "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set_value("user", bad_key, "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.add_value(bad_key, "hooksPath", "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.add_value("user", bad_key, "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.rename_section("user", bad_key) + + git_config.set_value("user", "name", "safe") + + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertEqual(git_config.get_value("user", "name"), "safe") + self.assertFalse(git_config.has_section("core")) + + @with_rw_directory + def test_writer_rejects_invalid_option_names(self, rw_dir): + config_path = osp.join(rw_dir, "config") + bad_options = ( + "name=value", + "name#comment", + "name;comment", + "name with space", + "name\twith-tab", + "name[section", + "name]section", + "name:colon", + 'name"quote', + "name\\escape", + ) + + with GitConfigParser(config_path, read_only=False) as git_config: + git_config.add_section("user") + for bad_option in bad_options: + with pytest.raises(ValueError, match="option name"): + git_config.set("user", bad_option, "unsafe") + with pytest.raises(ValueError, match="option name"): + git_config.set_value("user", bad_option, "unsafe") + with pytest.raises(ValueError, match="option name"): + git_config.add_value("user", bad_option, "unsafe") + + git_config.set_value("user", "safe-option1", "safe") + git_config.set_value("user", "safe_option2", "safe") + git_config.set_value("user", "3safe_option", "safe") + git_config.set_value("user", "safe.option3", "safe") + + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertEqual(git_config.get_value("user", "safe-option1"), "safe") + self.assertEqual(git_config.get_value("user", "safe_option2"), "safe") + self.assertEqual(git_config.get_value("user", "3safe_option"), "safe") + self.assertEqual(git_config.get_value("user", "safe.option3"), "safe") + + @with_rw_directory + def test_writer_rejects_unquoted_section_terminators(self, rw_dir): + config_path = osp.join(rw_dir, "config") + bad_sections = ( + "user] [other", + 'user"] [other"', + 'submodule "docs"] [other', + 'submodule "docs] [other', + 'submodule "docs] [other\\', + ) + safe_section = 'submodule "docs]archive"' + + with GitConfigParser(config_path, read_only=False) as git_config: + git_config.add_section("user") + for bad_section in bad_sections: + with pytest.raises(ValueError, match="section name"): + git_config.add_section(bad_section) + with pytest.raises(ValueError, match="section name"): + git_config.set(bad_section, "name", "value") + with pytest.raises(ValueError, match="section name"): + git_config.set_value(bad_section, "name", "value") + with pytest.raises(ValueError, match="section name"): + git_config.add_value(bad_section, "name", "value") + with pytest.raises(ValueError, match="section name"): + git_config.rename_section("user", bad_section) + + git_config.set_value("user", "name", "safe") + git_config.set_value(safe_section, "name", "safe") + self.assertEqual(git_config.get_value(safe_section, "name"), "safe") + + # A closing bracket inside a quoted subsection name is data, not a section terminator. + with open(config_path, "rb") as config_file: + self.assertIn( + b'[submodule "docs]archive"]\n', + config_file.read(), + "a closing bracket within a quoted subsection name should be preserved", + ) + + # Reparse the file to verify that rejected names did not inject an [other] section. + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertEqual( + git_config.get_value("user", "name"), + "safe", + "rejected section names corrupted the existing section", + ) + self.assertFalse(git_config.has_section("other"), "an unsafe section name injected an [other] section") + + @with_rw_directory + def test_set_and_add_value_reject_unsafe_value_characters(self, rw_dir): + config_path = osp.join(rw_dir, "config") + bad_values = ("foo\rbar", "foo\nbar", "foo\x00bar", b"foo\nbar") + + with GitConfigParser(config_path, read_only=False) as git_config: + git_config.add_section("user") + for bad_value in bad_values: + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set("user", "name", bad_value) + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set_value("user", "name", bad_value) + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.add_value("user", "name", bad_value) + + git_config.set_value("user", "name", "safe") + + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertEqual(git_config.get_value("user", "name"), "safe") + def test_base(self): path_repo = fixture_path("git_config") path_global = fixture_path("git_config_global") @@ -235,15 +505,81 @@ def check_test_value(cr, value): with GitConfigParser(fpa, read_only=True) as cr: check_test_value(cr, tv) - @pytest.mark.xfail( - os.name == "nt", - reason='Second config._has_includes() assertion fails (for "config is included if path is matching git_dir")', - raises=AssertionError, - ) + @with_rw_directory + def test_config_relative_path_include(self, rw_dir): + included_path = osp.join(rw_dir, "included") + with GitConfigParser(included_path, read_only=False) as cw: + cw.set_value("included", "value", "included") + + config_path = osp.join(rw_dir, "config") + with GitConfigParser(config_path, read_only=False) as cw: + cw.set_value("include", "path", "included") + + if osp.splitdrive(config_path)[0] != osp.splitdrive(os.getcwd())[0]: + pytest.skip("The temporary directory and checkout are on different drives") + + relative_config_path = osp.relpath(config_path) + with GitConfigParser(relative_config_path, read_only=True) as cr: + assert cr.get_value("included", "value") == "included" + + @pytest.mark.skipif(os.name != "nt", reason="Specifically for Windows drive-rooted paths.") + @with_rw_directory + def test_config_drive_rooted_path_include(self, rw_dir): + with cwd(rw_dir): + included_path = osp.join(rw_dir, "included") + with GitConfigParser(included_path, read_only=False) as cw: + cw.set_value("included", "value", "included") + + _drive, rooted_included_path = osp.splitdrive(included_path) + config_path = osp.join(rw_dir, "config") + with GitConfigParser(config_path, read_only=False) as cw: + cw.set_value("include", "path", rooted_included_path) + + with GitConfigParser(config_path, read_only=True) as cr: + assert cr.get_value("included", "value") == "included" + + @with_rw_directory + def test_multiple_include_paths_with_same_key(self, rw_dir): + """Test that multiple 'path' entries under [include] are all respected. + + Regression test for https://github.com/gitpython-developers/GitPython/issues/2099. + Git config allows multiple ``path`` values under ``[include]``, e.g.:: + + [include] + path = file1 + path = file2 + + Previously only one of these was included because _OMD.items() returns + only the last value for each key. + """ + # Create two config files to be included. + fp_inc1 = osp.join(rw_dir, "inc1.cfg") + fp_inc2 = osp.join(rw_dir, "inc2.cfg") + fp_main = osp.join(rw_dir, "main.cfg") + + with GitConfigParser(fp_inc1, read_only=False) as cw: + cw.set_value("user", "name", "from-inc1") + + with GitConfigParser(fp_inc2, read_only=False) as cw: + cw.set_value("core", "bar", "from-inc2") + + # Write a config with two path entries under a single [include] section. + # We write it manually because set_value would overwrite the key. + with open(fp_main, "w") as f: + f.write("[include]\n") + f.write(f"\tpath = {fp_inc1}\n") + f.write(f"\tpath = {fp_inc2}\n") + + with GitConfigParser(fp_main, read_only=True) as cr: + # Both included files should be loaded. + assert cr.get_value("user", "name") == "from-inc1" + assert cr.get_value("core", "bar") == "from-inc2" + @with_rw_directory def test_conditional_includes_from_git_dir(self, rw_dir): # Initiate repository path. git_dir = osp.join(rw_dir, "target1", "repo1") + git_dir_pattern = git_dir.replace("\\", "/") os.makedirs(git_dir) # Initiate mocked repository. @@ -255,6 +591,7 @@ def test_conditional_includes_from_git_dir(self, rw_dir): template = '[includeIf "{}:{}"]\n path={}\n' with open(path1, "w") as stream: + # on Windows, this writes a backslash pattern. stream.write(template.format("gitdir", git_dir, path2)) # Ensure that config is ignored if no repo is set. @@ -262,14 +599,26 @@ def test_conditional_includes_from_git_dir(self, rw_dir): assert not config._has_includes() assert config._included_paths() == [] - # Ensure that config is included if path is matching git_dir. + # Git uses forward slashes in gitdir patterns on every platform: + # backslashes escape the next pattern character rather than separate + # path components. On Windows, GitPython therefore normalizes git_dir + # to forward slashes but leaves this backslash pattern unchanged, so + # the two do not match and no path is included. + with GitConfigParser(path1, repo=repo, merge_includes=False) as config: + expected_paths = [] if sys.platform == "win32" else [("path", path2)] + assert config._included_paths() == expected_paths + + # Ensure that Git's forward-slash syntax matches native Windows paths. + with open(path1, "w") as stream: + stream.write(template.format("gitdir", git_dir_pattern, path2)) + with GitConfigParser(path1, repo=repo) as config: assert config._has_includes() assert config._included_paths() == [("path", path2)] # Ensure that config is ignored if case is incorrect. with open(path1, "w") as stream: - stream.write(template.format("gitdir", git_dir.upper(), path2)) + stream.write(template.format("gitdir", git_dir_pattern.upper(), path2)) with GitConfigParser(path1, repo=repo) as config: assert not config._has_includes() @@ -277,7 +626,7 @@ def test_conditional_includes_from_git_dir(self, rw_dir): # Ensure that config is included if case is ignored. with open(path1, "w") as stream: - stream.write(template.format("gitdir/i", git_dir.upper(), path2)) + stream.write(template.format("gitdir/i", git_dir_pattern.upper(), path2)) with GitConfigParser(path1, repo=repo) as config: assert config._has_includes() @@ -307,6 +656,20 @@ def test_conditional_includes_from_git_dir(self, rw_dir): assert config._has_includes() assert config._included_paths() == [("path", path2)] + @with_rw_directory + def test_conditional_includes_do_not_treat_backslashes_as_separators(self, rw_dir): + git_dir = osp.join(rw_dir, "target", "repo") + repo = mock.Mock(git_dir=git_dir) + config_path = osp.join(rw_dir, "config") + included_path = osp.join(rw_dir, "included") + pattern = git_dir.replace("\\", "/").replace("/target/repo", R"/target\repo") + + with open(config_path, "w") as stream: + stream.write(f'[includeIf "gitdir:{pattern}"]\n path={included_path}\n') + + with GitConfigParser(config_path, repo=repo, merge_includes=False) as config: + assert config._included_paths() == [] + @with_rw_directory def test_conditional_includes_from_branch_name(self, rw_dir): # Initiate mocked branch. @@ -362,6 +725,41 @@ def test_conditional_includes_from_branch_name_error(self, rw_dir): assert not config._has_includes() assert config._included_paths() == [] + @with_rw_directory + def test_conditional_includes_remote_url(self, rw_dir): + # Initiate mocked repository. + repo = mock.Mock() + repo.remotes = [mock.Mock(url="https://github.com/foo/repo")] + + # Initiate config files. + path1 = osp.join(rw_dir, "config1") + path2 = osp.join(rw_dir, "config2") + template = '[includeIf "hasconfig:remote.*.url:{}"]\n path={}\n' + + # Ensure that config with hasconfig and full url is correct. + with open(path1, "w") as stream: + stream.write(template.format("https://github.com/foo/repo", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert config._has_includes() + assert config._included_paths() == [("path", path2)] + + # Ensure that config with hasconfig and incorrect url is incorrect. + with open(path1, "w") as stream: + stream.write(template.format("incorrect", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert not config._has_includes() + assert config._included_paths() == [] + + # Ensure that config with hasconfig and url using glob pattern is correct. + with open(path1, "w") as stream: + stream.write(template.format("**/**github.com*/**", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert config._has_includes() + assert config._included_paths() == [("path", path2)] + def test_rename(self): file_obj = self._to_memcache(fixture_path("git_config")) with GitConfigParser(file_obj, read_only=False, merge_includes=False) as cw: @@ -380,13 +778,17 @@ def test_complex_aliases(self): with GitConfigParser(file_obj, read_only=False) as w_config: self.assertEqual( w_config.get("alias", "rbi"), - '"!g() { git rebase -i origin/${1:-master} ; } ; g"', + "!g() { git rebase -i origin/${1:-master} ; } ; g", ) self.assertEqual( file_obj.getvalue(), self._to_memcache(fixture_path(".gitconfig")).getvalue(), ) + def test_config_with_extra_whitespace(self): + cr = GitConfigParser(fixture_path("git_config_with_extra_whitespace"), read_only=True) + self.assertEqual(cr.get("init", "defaultBranch"), "trunk") + def test_empty_config_value(self): cr = GitConfigParser(fixture_path("git_config_with_empty_value"), read_only=True) @@ -395,6 +797,43 @@ def test_empty_config_value(self): with self.assertRaises(cp.NoOptionError): cr.get_value("color", "ui") + def test_config_with_quotes(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes"), read_only=True) + + self.assertEqual(cr.get("user", "name"), "Cody Veal") + self.assertEqual(cr.get("user", "email"), "cveal05@gmail.com") + + def test_config_with_empty_quotes(self): + cr = GitConfigParser(fixture_path("git_config_with_empty_quotes"), read_only=True) + self.assertEqual(cr.get("core", "filemode"), "", "quotes can form a literal empty string as value") + + def test_config_with_quotes_with_literal_whitespace(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace_inside"), read_only=True) + self.assertEqual(cr.get("core", "commentString"), "# ") + + def test_config_with_quotes_with_whitespace_outside_value(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace_outside"), read_only=True) + self.assertEqual(cr.get("init", "defaultBranch"), "trunk") + + def test_config_with_quotes_containing_escapes(self): + """Interpret Git's quoted escapes without changing malformed values.""" + cr = GitConfigParser(fixture_path("git_config_with_quotes_escapes"), read_only=True) + + self.assertEqual(cr.get("custom", "hasnewline"), "first\nsecond") + self.assertEqual(cr.get("custom", "hasbackslash"), R"foo\bar") + self.assertEqual(cr.get("custom", "hasquote"), 'ab"cd') + self.assertEqual(cr.get("custom", "hastrailingbackslash"), "word\\") + self.assertEqual(cr.get("custom", "hasunrecognized"), R"p\qrs") + + # It is less obvious whether and what to eventually do with this. + self.assertEqual(cr.get("custom", "hasunescapedquotes"), '"ab"cd"e"') + + # Cases where quote removal is clearly safe should happen even after those. + self.assertEqual(cr.get("custom", "ordinary"), "hello world") + + # Cases without quotes should still parse correctly even after those, too. + self.assertEqual(cr.get("custom", "unquoted"), "good evening") + def test_get_values_works_without_requiring_any_other_calls_first(self): file_obj = self._to_memcache(fixture_path("git_config_multiple")) cr = GitConfigParser(file_obj, read_only=True) diff --git a/test/test_db.py b/test/test_db.py index de093cbd8..46580d84b 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -3,15 +3,30 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +from io import BytesIO +import os.path as osp +from unittest import mock + +from gitdb import IStream +from gitdb.db import LooseObjectDB +from gitdb.typ import str_blob_type + from git.db import GitCmdObjectDB from git.exc import BadObject -from test.lib import TestBase from git.util import bin_to_hex -import os.path as osp +from test.lib import TestBase, with_rw_repo class TestDB(TestBase): + @with_rw_repo("HEAD") + def test_store_uses_hash_object(self, rw_repo): + data = b"hello world" + with mock.patch.object(LooseObjectDB, "store", side_effect=AssertionError("unexpected loose-object write")): + istream = rw_repo.odb.store(IStream(str_blob_type, len(data), BytesIO(data))) + + assert rw_repo.odb.stream(istream.binsha).read() == data + def test_base(self): gdb = GitCmdObjectDB(osp.join(self.rorepo.git_dir, "objects"), self.rorepo.git) diff --git a/test/test_diff.py b/test/test_diff.py index 87f92f5d1..9cfbffd17 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -4,16 +4,19 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import gc -import os import os.path as osp import shutil +import sys import tempfile import ddt import pytest -from git import NULL_TREE, Diff, DiffIndex, GitCommandError, Repo, Submodule +from git import NULL_TREE, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule from git.cmd import Git +from git.diff import decode_path +from git.exc import UnsafeOptionError + from test.lib import StringProcessAdapter, TestBase, fixture, with_rw_directory @@ -90,6 +93,26 @@ def test_diff_with_staged_file(self, rw_dir): "This should work, but doesn't right now ... it's OK", ) + @with_rw_directory + def test_patch_diff_ignores_mnemonic_prefix(self, rw_dir): + repo = Repo.init(rw_dir) + filepath = osp.join(rw_dir, "file.txt") + with open(filepath, "w") as stream: + stream.write("initial\n") + repo.index.add([filepath]) + repo.index.commit("initial") + + with open(filepath, "w") as stream: + stream.write("changed\n") + repo.git.config("diff.mnemonicPrefix", "true") + + self.assertEqual(len(repo.head.commit.diff(None, create_patch=True)), 1) + self.assertEqual(len(repo.index.diff(None, create_patch=True)), 1) + + repo.index.add([filepath]) + self.assertEqual(len(repo.index.diff(NULL_TREE, create_patch=True)), 1) + self.assertEqual(repo.git.config("--get", "diff.mnemonicPrefix"), "true") + def test_list_from_string_new_mode(self): output = StringProcessAdapter(fixture("diff_new_mode")) diffs = Diff._index_from_patch_format(self.rorepo, output) @@ -271,18 +294,18 @@ def test_diff_unsafe_paths(self): self.assertEqual(res[10].b_rawpath, b"path/\x80-invalid-unicode-path.txt") # The "Moves" - # NOTE: The path prefixes a/ and b/ here are legit! We're actually - # verifying that it's not "a/a/" that shows up, see the fixture data. - self.assertEqual(res[11].a_path, "a/with spaces") # NOTE: path a/ here legit! - self.assertEqual(res[11].b_path, "b/with some spaces") # NOTE: path b/ here legit! + # NOTE: The path prefixes "a/" and "b/" here are legit! We're actually verifying + # that it's not "a/a/" that shows up; see the fixture data. + self.assertEqual(res[11].a_path, "a/with spaces") # NOTE: path "a/"" legit! + self.assertEqual(res[11].b_path, "b/with some spaces") # NOTE: path "b/"" legit! self.assertEqual(res[12].a_path, "a/ending in a space ") self.assertEqual(res[12].b_path, "b/ending with space ") self.assertEqual(res[13].a_path, 'a/"with-quotes"') self.assertEqual(res[13].b_path, 'b/"with even more quotes"') def test_diff_patch_format(self): - # Test all of the 'old' format diffs for completeness - it should at least - # be able to deal with it. + # Test all of the 'old' format diffs for completeness - it should at least be + # able to deal with it. fixtures = ( "diff_2", "diff_2f", @@ -302,6 +325,11 @@ def test_diff_patch_format(self): Diff._index_from_patch_format(self.rorepo, diff_proc) # END for each fixture + def test_decode_path_distinguishes_escaped_backslashes_from_octal_bytes(self): + self.assertEqual(decode_path(b'"foo\\\\899bar"', False), b"foo\\899bar") + self.assertEqual(decode_path(b'"foo\\\\123bar"', False), b"foo\\123bar") + self.assertEqual(decode_path(b'"foo\\123bar"', False), b"fooSbar") + def test_diff_with_spaces(self): data = StringProcessAdapter(fixture("diff_file_with_spaces")) diff_index = Diff._index_from_patch_format(self.rorepo, data) @@ -309,7 +337,7 @@ def test_diff_with_spaces(self): self.assertEqual(diff_index[0].b_path, "file with spaces", repr(diff_index[0].b_path)) @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason='"Access is denied" when tearDown calls shutil.rmtree', raises=PermissionError, ) @@ -345,13 +373,58 @@ def test_diff_submodule(self): repo.create_tag("2") diff = repo.commit("1").diff(repo.commit("2"))[0] - # If diff is unable to find the commit hashes (looks in wrong repo) the *_blob.size - # property will be a string containing exception text, an int indicates success. + # If diff is unable to find the commit hashes (looks in wrong repo) the + # *_blob.size property will be a string containing exception text, an int + # indicates success. self.assertIsInstance(diff.a_blob.size, int) self.assertIsInstance(diff.b_blob.size, int) + def test_diff_rejects_unsafe_output_options(self): + commit = self.rorepo.head.commit + + commit.diff(S="needle") + + calls = ( + lambda target: commit.diff(output=target), + lambda target: commit.diff(other=f"--output={target}"), + lambda target: commit.diff(O=target), + lambda target: commit.diff(orderfile=target), + lambda target: commit.diff(other=f"--orderfile={target}"), + lambda target: commit.diff(other=f"-pO{target}"), + lambda target: commit.diff(other=f"-uO{target}"), + lambda target: commit.diff(other=f"-DO{target}"), + lambda target: self.rorepo.index.diff(NULL_TREE, output=target), + lambda target: self.rorepo.index.diff(f"--output={target}"), + lambda target: self.rorepo.index.diff(NULL_TREE, O=target), + lambda target: self.rorepo.index.diff(NULL_TREE, orderfile=target), + lambda target: self.rorepo.index.diff(f"--orderfile={target}"), + lambda target: self.rorepo.index.diff(f"-pO{target}"), + lambda target: self.rorepo.index.diff(f"-uO{target}"), + lambda target: self.rorepo.index.diff(f"-DO{target}"), + ) + for index, call in enumerate(calls): + target = osp.join(self.repo_dir, f"diff-output-{index}") + with self.assertRaises(UnsafeOptionError): + call(target) + self.assertFalse(osp.exists(target)) + + allowed_target = osp.join(self.repo_dir, "allowed-diff-output") + commit.diff(output=allowed_target, allow_unsafe_options=True) + self.assertTrue(osp.isfile(allowed_target)) + + def test_diff_rejects_no_index(self): + calls = ( + lambda: self.rorepo.head.commit.diff(no_index=True), + lambda: self.rorepo.head.commit.diff(other="--no-index"), + lambda: self.rorepo.index.diff(None, no_index=True), + lambda: self.rorepo.index.diff("--no-index"), + ) + for call in calls: + with self.assertRaises(UnsafeOptionError): + call() + def test_diff_interface(self): - # Test a few variations of the main diff routine. + """Test a few variations of the main diff routine.""" assertion_map = {} for i, commit in enumerate(self.rorepo.iter_commits("0.1.6", max_count=2)): diff_item = commit @@ -359,7 +432,7 @@ def test_diff_interface(self): diff_item = commit.tree # END use tree every second item - for other in (None, NULL_TREE, commit.Index, commit.parents[0]): + for other in (None, NULL_TREE, commit.INDEX, commit.parents[0]): for paths in (None, "CHANGES", ("CHANGES", "lib")): for create_patch in range(2): diff_index = diff_item.diff(other=other, paths=paths, create_patch=create_patch) @@ -392,9 +465,9 @@ def test_diff_interface(self): # END for each other side # END for each commit - # Assert that we could always find at least one instance of the members we - # can iterate in the diff index - if not this indicates its not working correctly - # or our test does not span the whole range of possibilities. + # Assert that we could always find at least one instance of the members we can + # iterate in the diff index - if not this indicates its not working correctly or + # our test does not span the whole range of possibilities. for key, value in assertion_map.items(): self.assertIsNotNone(value, "Did not find diff for %s" % key) # END for each iteration type @@ -405,10 +478,22 @@ def test_diff_interface(self): diff_index = c.diff(cp, ["does/not/exist"]) self.assertEqual(len(diff_index), 0) + def test_diff_interface_stability(self): + """Test that the Diffable.Index redefinition should not break compatibility.""" + self.assertIs( + Diffable.Index, + Diffable.INDEX, + "The old and new class attribute names must be aliases.", + ) + self.assertIs( + type(Diffable.INDEX).__eq__, + object.__eq__, + "Equality comparison must be reference-based.", + ) + @with_rw_directory def test_rename_override(self, rw_dir): - """Test disabling of diff rename detection""" - + """Test disabling of diff rename detection.""" # Create and commit file_a.txt. repo = Repo.init(rw_dir) file_a = osp.join(rw_dir, "file_a.txt") @@ -473,3 +558,64 @@ def test_rename_override(self, rw_dir): self.assertEqual(True, diff.renamed_file) self.assertEqual("file_a.txt", diff.rename_from) self.assertEqual("file_b.txt", diff.rename_to) + + @with_rw_directory + def test_diff_patch_with_external_engine(self, rw_dir): + repo = Repo.init(rw_dir) + gitignore = osp.join(rw_dir, ".gitignore") + + # First commit + with open(gitignore, "w") as f: + f.write("first_line\n") + repo.git.add(".gitignore") + repo.index.commit("first commit") + + # Adding second line and committing + with open(gitignore, "a") as f: + f.write("second_line\n") + repo.git.add(".gitignore") + repo.index.commit("second commit") + + # Adding third line and staging + with open(gitignore, "a") as f: + f.write("third_line\n") + repo.git.add(".gitignore") + + # Adding fourth line + with open(gitignore, "a") as f: + f.write("fourth_line\n") + + # Set the external diff engine + with repo.config_writer(config_level="repository") as writer: + writer.set_value("diff", "external", "bogus_diff_engine") + + head_against_head = repo.head.commit.diff("HEAD^", create_patch=True) + self.assertEqual(len(head_against_head), 1) + head_against_index = repo.head.commit.diff(create_patch=True) + self.assertEqual(len(head_against_index), 1) + head_against_working_tree = repo.head.commit.diff(None, create_patch=True) + self.assertEqual(len(head_against_working_tree), 1) + + index_against_head = repo.index.diff("HEAD", create_patch=True) + self.assertEqual(len(index_against_head), 1) + index_against_working_tree = repo.index.diff(None, create_patch=True) + self.assertEqual(len(index_against_working_tree), 1) + + @with_rw_directory + def test_beginning_space(self, rw_dir): + # Create a file beginning by a whitespace + repo = Repo.init(rw_dir) + file = osp.join(rw_dir, " file.txt") + with open(file, "w") as f: + f.write("hello world") + repo.git.add(Git.polish_url(file)) + repo.index.commit("first commit") + + # Diff the commit with an empty tree + # and check the paths + diff_index = repo.head.commit.diff(NULL_TREE) + d = diff_index[0] + a_path = d.a_path + b_path = d.b_path + self.assertEqual(a_path, " file.txt") + self.assertEqual(b_path, " file.txt") diff --git a/test/test_docs.py b/test/test_docs.py index 48922eba8..0810d954d 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -5,22 +5,19 @@ import gc import os -import sys - -import pytest +import os.path from test.lib import TestBase -from test.lib.helper import with_rw_directory - -import os.path +from test.lib.helper import with_rw_directory, with_rw_repo class Tutorials(TestBase): def tearDown(self): gc.collect() - # ACTUALLY skipped by git.util.rmtree (in local onerror function), from the last call to it via - # git.objects.submodule.base.Submodule.remove (at "handle separate bare repository"), line 1062. + # ACTUALLY skipped by git.util.rmtree (in local onerror function), from the last + # call to it via git.objects.submodule.base.Submodule.remove + # (at "handle separate bare repository"), line 1062. # # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: helper.wrapper fails with: PermissionError: [WinError 5] Access is denied: " @@ -31,8 +28,8 @@ def test_init_repo_object(self, rw_dir): from git import Repo # rorepo is a Repo instance pointing to the git-python repository. - # For all you know, the first argument to Repo is a path to the repository - # you want to work with. + # For all you know, the first argument to Repo is a path to the repository you + # want to work with. repo = Repo(self.rorepo.working_tree_dir) assert not repo.bare # ![1-test_init_repo_object] @@ -75,21 +72,20 @@ def test_init_repo_object(self, rw_dir): # heads, tags and references # heads are branches in git-speak # [8-test_init_repo_object] - self.assertEqual( - repo.head.ref, - repo.heads.master, # head is a sym-ref pointing to master. - "It's ok if TC not running from `master`.", - ) + active_branch = repo.active_branch + self.assertEqual(repo.head.ref, active_branch) # HEAD is a sym-ref pointing to the active branch. self.assertEqual(repo.tags["0.3.5"], repo.tag("refs/tags/0.3.5")) # You can access tags in various ways too. - self.assertEqual(repo.refs.master, repo.heads["master"]) # .refs provides all refs, i.e. heads... + self.assertEqual(repo.refs[active_branch.name], repo.heads[active_branch.name]) # .refs provides all refs... if "TRAVIS" not in os.environ: - self.assertEqual(repo.refs["origin/master"], repo.remotes.origin.refs.master) # ... remotes ... + remote_branch = next(ref for ref in repo.remotes.origin.refs if ref.remote_head != "HEAD") + self.assertEqual(repo.refs[remote_branch.name], remote_branch) # ...remotes... self.assertEqual(repo.refs["0.3.5"], repo.tags["0.3.5"]) # ... and tags. # ![8-test_init_repo_object] # Create a new head/branch. # [9-test_init_repo_object] + original_branch = cloned_repo.active_branch new_branch = cloned_repo.create_head("feature") # Create a new branch ... assert cloned_repo.active_branch != new_branch # which wasn't checked out yet ... self.assertEqual(new_branch.commit, cloned_repo.active_branch.commit) # pointing to the checked-out commit. @@ -149,10 +145,10 @@ def update(self, op_code, cur_count, max_count=None, message=""): assert origin.exists() for fetch_info in origin.fetch(progress=MyProgressPrinter()): print("Updated %s to %s" % (fetch_info.ref, fetch_info.commit)) - # Create a local branch at the latest fetched master. We specify the name statically, but you have all - # information to do it programmatically as well. - bare_master = bare_repo.create_head("master", origin.refs.master) - bare_repo.head.set_reference(bare_master) + # Create a local branch at one of the remote's branches. + remote_branch = next(ref for ref in origin.refs if ref.remote_head != "HEAD") + bare_branch = bare_repo.create_head(remote_branch.remote_head, remote_branch) + bare_repo.head.set_reference(bare_branch) assert not bare_repo.delete_remote(origin).exists() # push and pull behave very similarly. # ![12-test_init_repo_object] @@ -165,35 +161,39 @@ def update(self, op_code, cur_count, max_count=None, message=""): new_file_path = os.path.join(cloned_repo.working_tree_dir, "my-new-file") open(new_file_path, "wb").close() # Create new file in working tree. cloned_repo.index.add([new_file_path]) # Add it to the index. - # Commit the changes to deviate masters history. + # Commit the changes to deviate from the original branch's history. cloned_repo.index.commit("Added a new file in the past - for later merge") # Prepare a merge. - master = cloned_repo.heads.master # Right-hand side is ahead of us, in the future. - merge_base = cloned_repo.merge_base(new_branch, master) # Allows for a three-way merge. - cloned_repo.index.merge_tree(master, base=merge_base) # Write the merge result into index. + merge_base = cloned_repo.merge_base(new_branch, original_branch) # Allows for a three-way merge. + cloned_repo.index.merge_tree(original_branch, base=merge_base) # Write the merge result into index. cloned_repo.index.commit( "Merged past and now into future ;)", - parent_commits=(new_branch.commit, master.commit), + parent_commits=(new_branch.commit, original_branch.commit), ) - # Now new_branch is ahead of master, which probably should be checked out and reset softly. + # Now new_branch is ahead of the original branch, which probably should be checked out and reset softly. # Note that all these operations didn't touch the working tree, as we managed it ourselves. # This definitely requires you to know what you are doing! :) assert os.path.basename(new_file_path) in new_branch.commit.tree # New file is now in tree. - master.commit = new_branch.commit # Let master point to most recent commit. - cloned_repo.head.reference = master # We adjusted just the reference, not the working tree or index. + original_branch.commit = new_branch.commit # Let the original branch point to the most recent commit. + cloned_repo.head.reference = original_branch # We adjusted just the reference, not the working tree or index. # ![13-test_init_repo_object] # submodules # [14-test_init_repo_object] - # Create a new submodule and check it out on the spot, setup to track master branch of `bare_repo`. - # As our GitPython repository has submodules already that point to GitHub, make sure we don't - # interact with them. + # Create a new submodule and check it out on the spot, set up to track the + # default branch of `bare_repo`. As our GitPython repository has submodules + # already that point to GitHub, make sure we don't interact with them. for sm in cloned_repo.submodules: assert not sm.remove().exists() # after removal, the sm doesn't exist anymore - sm = cloned_repo.create_submodule("mysubrepo", "path/to/subrepo", url=bare_repo.git_dir, branch="master") + sm = cloned_repo.create_submodule( + "mysubrepo", + "path/to/subrepo", + url=bare_repo.git_dir, + branch=bare_branch.name, + ) # .gitmodules was written and added to the index, which is now being committed. cloned_repo.index.commit("Added submodule") @@ -424,8 +424,8 @@ def test_references_and_objects(self, rw_dir): with origin.config_writer as cw: cw.set("pushurl", "other_url") - # Please note that in Python 2, writing origin.config_writer.set(...) is totally safe. - # In py3 __del__ calls can be delayed, thus not writing changes in time. + # Please note that in Python 2, writing origin.config_writer.set(...) is totally + # safe. In py3 __del__ calls can be delayed, thus not writing changes in time. # ![26-test_references_and_objects] # [27-test_references_and_objects] @@ -462,33 +462,36 @@ def test_references_and_objects(self, rw_dir): # ![29-test_references_and_objects] # [30-test_references_and_objects] - # Check out the branch using git-checkout. It will fail as the working tree appears dirty. + # Check out the branch using git-checkout. + # It will fail as the working tree appears dirty. self.assertRaises(git.GitCommandError, repo.heads.master.checkout) repo.heads.past_branch.checkout() # ![30-test_references_and_objects] # [31-test_references_and_objects] - git = repo.git - git.checkout("HEAD", b="my_new_branch") # Create a new branch. - git.branch("another-new-one") - git.branch("-D", "another-new-one") # Pass strings for full control over argument order. - git.for_each_ref() # '-' becomes '_' when calling it. + git_cmd = repo.git + git_cmd.checkout("HEAD", b="my_new_branch") # Create a new branch. + git_cmd.branch("another-new-one") + git_cmd.branch("-D", "another-new-one") # Pass strings for full control over argument order. + git_cmd.for_each_ref() # '-' becomes '_' when calling it. # ![31-test_references_and_objects] repo.git.clear_cache() - @pytest.mark.xfail( - sys.platform == "cygwin", - reason="Cygwin GitPython can't find SHA for submodule", - raises=ValueError, - ) - def test_submodules(self): + @with_rw_repo("c15a6e1923a14bc760851913858a3942a4193cdb") + def test_submodules(self, repo): # [1-test_submodules] - repo = self.rorepo sms = repo.submodules assert len(sms) == 1 sm = sms[0] + with sm.config_writer() as writer: + writer.set_value("url", self._gitdb_repo_url()) + sm.update(recursive=False) + child = sm.children()[0] + with child.config_writer() as writer: + writer.set_value("url", self._small_repo_url()) + child.update() self.assertEqual(sm.name, "gitdb") # GitPython has gitdb as its one and only (direct) submodule... self.assertEqual(sm.children()[0].name, "smmap") # ...which has smmap as its one and only submodule. diff --git a/test/test_exc.py b/test/test_exc.py index 3f4d0b803..2e979f5a1 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -3,9 +3,11 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +from itertools import product import re import ddt + from git.exc import ( InvalidGitRepositoryError, WorkTreeRepositoryUnsupported, @@ -20,9 +22,8 @@ RepositoryDirtyError, ) from git.util import remove_password_if_present -from test.lib import TestBase -import itertools as itt +from test.lib import TestBase _cmd_argvs = ( @@ -51,7 +52,7 @@ _streams_n_substrings = ( None, - "steram", + "stream", "ÎŋÎŧÎŋĪĪ†Îŋ stream", ) @@ -79,7 +80,7 @@ def test_ExceptionsHaveBaseClass(self): for ex_class in exception_classes: self.assertTrue(issubclass(ex_class, GitError)) - @ddt.data(*list(itt.product(_cmd_argvs, _causes_n_substrings, _streams_n_substrings))) + @ddt.data(*list(product(_cmd_argvs, _causes_n_substrings, _streams_n_substrings))) def test_CommandError_unicode(self, case): argv, (cause, subs), stream = case cls = CommandError diff --git a/test/test_fixture_health.py b/test/test_fixture_health.py new file mode 100644 index 000000000..8301001c4 --- /dev/null +++ b/test/test_fixture_health.py @@ -0,0 +1,70 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Verify that the source repository is usable by git.""" + +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent + +FIXTURE_DIRS = [pytest.param(REPO_ROOT, id="repo_root")] + + +def test_source_tree_has_no_gitlinks() -> None: + """All formerly external dependencies are regular monorepo directories.""" + if not (REPO_ROOT / ".git").exists(): + pytest.skip(f"{REPO_ROOT} is not a git checkout") + try: + result = subprocess.run( + ["git", "ls-files", "--stage"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + except FileNotFoundError: + pytest.skip("git is not installed or not on PATH") + gitlinks = [line for line in result.stdout.splitlines() if line.startswith("160000 ")] + assert not gitlinks, "Unexpected submodules:\n" + "\n".join(gitlinks) + + +@pytest.mark.parametrize("fixture_dir", FIXTURE_DIRS) +def test_fixture_dir_is_trusted_by_git(fixture_dir: Path) -> None: + """git accepts ``fixture_dir`` as its own repository owned by a trusted user. + + Run ``git -C rev-parse --show-toplevel`` and assert it + succeeds and reports ``fixture_dir`` itself as the toplevel. Failure + typically means the directory's on-disk ownership doesn't match the + running user and the CI workflow's ``safe.directory`` list is missing + an entry that would override the check. + """ + if not (fixture_dir / ".git").exists(): + pytest.skip(f"{fixture_dir} is not a git checkout") + try: + result = subprocess.run( + ["git", "-C", str(fixture_dir), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError: + pytest.skip("git is not installed or not on PATH") + assert result.returncode == 0, ( + f"git refuses to operate in {fixture_dir}.\n" + f"stderr: {result.stderr.strip()}\n" + "The directory's owner doesn't match the running user and no " + "`safe.directory` entry overrides the check. On CI, the " + "workflow's `safe.directory` list typically needs an entry for " + "this path. Locally, this is unexpected and usually indicates " + "an ownership problem worth investigating." + ) + reported = Path(result.stdout.strip()) + assert reported.samefile(fixture_dir), ( + f"git reports the toplevel as {reported}, " + f"not as {fixture_dir} itself. " + "This usually means the directory is not an initialized git " + "repository (its `.git` marker may be stale or pointing elsewhere)." + ) diff --git a/test/test_fun.py b/test/test_fun.py index 566bc9aae..38f506ff4 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -2,27 +2,26 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO -from stat import S_IFDIR, S_IFREG, S_IFLNK, S_IXUSR +from stat import S_IFDIR, S_IFLNK, S_IFREG, S_IXUSR from os import stat import os.path as osp +from gitdb.base import IStream +from gitdb.typ import str_tree_type + from git import Git from git.index import IndexFile -from git.index.fun import ( - aggressive_tree_merge, - stat_mode_to_index_mode, -) +from git.index.fun import aggressive_tree_merge, stat_mode_to_index_mode from git.objects.fun import ( traverse_tree_recursive, traverse_trees_recursive, - tree_to_stream, tree_entries_from_data, + tree_to_stream, ) from git.repo.fun import find_worktree_git_dir -from test.lib import TestBase, with_rw_repo, with_rw_directory from git.util import bin_to_hex, cygpath, join_path_native -from gitdb.base import IStream -from gitdb.typ import str_tree_type + +from test.lib import TestBase, with_rw_directory, with_rw_repo class TestFun(TestBase): @@ -35,7 +34,8 @@ def _assert_index_entries(self, entries, trees): # END assert entry matches fully def test_aggressive_tree_merge(self): - # Head tree with additions, removals and modification compared to its predecessor. + # Head tree with additions, removals and modification compared to its + # predecessor. odb = self.rorepo.odb HC = self.rorepo.commit("6c1faef799095f3990e9970bc2cb10aa0221cf9c") H = HC.tree @@ -243,6 +243,7 @@ def test_tree_traversal(self): B_old = self.rorepo.tree("1f66cfbbce58b4b552b041707a12d437cc5f400a") # old base tree # Two very different trees. + entries = traverse_trees_recursive(odb, [B_old.binsha, H.binsha], "") self._assert_tree_entries(entries, 2) @@ -251,7 +252,10 @@ def test_tree_traversal(self): self._assert_tree_entries(oentries, 2) # Single tree. - is_no_tree = lambda i, d: i.type != "tree" + + def is_no_tree(i, _d): + return i.type != "tree" + entries = traverse_trees_recursive(odb, [B.binsha], "") assert len(entries) == len(list(B.traverse(predicate=is_no_tree))) self._assert_tree_entries(entries, 1) @@ -302,3 +306,7 @@ def test_linked_worktree_traversal(self, rw_dir): def test_tree_entries_from_data_with_failing_name_decode_py3(self): r = tree_entries_from_data(b"100644 \x9f\0aaa") assert r == [(b"aaa", 33188, "\udc9f")], r + + def test_tree_entries_from_bytearray(self): + r = tree_entries_from_data(bytearray(b"100644 name\0abcdefghijklmnopqrst")) + assert r == [(b"abcdefghijklmnopqrst", 33188, "name")], r diff --git a/test/test_git.py b/test/test_git.py index 3b9abc712..a88d980fb 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -6,15 +6,18 @@ import contextlib import gc import inspect +import io import logging import os import os.path as osp from pathlib import Path +import pickle import re import shutil import subprocess import sys -from tempfile import TemporaryFile +import tempfile +import time from unittest import skipUnless if sys.version_info >= (3, 8): @@ -24,8 +27,10 @@ import ddt -from git import Git, refresh, GitCommandError, GitCommandNotFound, Repo, cmd +from git import Git, GitCommandError, GitCommandNotFound, Repo, cmd, refresh +from git.exc import UnsafeOptionError from git.util import cwd, finalize_process + from test.lib import TestBase, fixture_path, with_rw_directory @@ -44,6 +49,55 @@ def _patch_out_env(name): os.environ[name] = old_value +@contextlib.contextmanager +def _rollback_refresh(): + old_git_executable = Git.GIT_PYTHON_GIT_EXECUTABLE + + if old_git_executable is None: + raise RuntimeError("no executable string (need initial refresh before test)") + + try: + yield old_git_executable # Provide the old value for convenience. + finally: + # The cleanup refresh should always raise an exception if it fails, since if it + # fails then previously discovered test results could be misleading and, more + # importantly, subsequent tests may be unable to run or give misleading results. + # So pre-set a non-None value, so that the cleanup will be a "second" refresh. + # This covers cases where a test has set it to None to test a "first" refresh. + Git.GIT_PYTHON_GIT_EXECUTABLE = Git.git_exec_name + + # Do the cleanup refresh. This sets Git.GIT_PYTHON_GIT_EXECUTABLE to old_value + # in most cases. The reason to call it is to achieve other associated state + # changes as well, which include updating attributes of the FetchInfo class. + refresh() + + +@contextlib.contextmanager +def _fake_git(*version_info): + fake_version = ".".join(map(str, version_info)) + fake_output = f"git version {fake_version} (fake)" + + with tempfile.TemporaryDirectory() as tdir: + if sys.platform == "win32": + fake_git = Path(tdir, "fake-git.cmd") + script = f"@echo {fake_output}\n" + fake_git.write_text(script, encoding="utf-8") + else: + fake_git = Path(tdir, "fake-git") + script = f"#!/bin/sh\necho '{fake_output}'\n" + fake_git.write_text(script, encoding="utf-8") + fake_git.chmod(0o755) + + yield str(fake_git.absolute()) + + +def _rename_with_stem(path, new_stem): + if sys.version_info >= (3, 9): + path.rename(path.with_stem(new_stem)) + else: + path.rename(path.with_name(new_stem + path.suffix)) + + @ddt.ddt class TestGit(TestBase): @classmethod @@ -102,6 +156,102 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): res = self.git.transform_kwargs(**{"s": True, "t": True}) self.assertEqual({"-s", "-t"}, set(res)) + def test_check_unsafe_options_normalizes_kwargs(self): + cases = [ + (["upload_pack"], ["--upload-pack"]), + (["receive_pack"], ["--receive-pack"]), + (["exec"], ["--exec"]), + (["u"], ["-u"]), + (["c"], ["-c"]), + (["--upload-pack=/tmp/helper"], ["--upload-pack"]), + (["--config core.filemode=false"], ["--config"]), + (["--upl=/tmp/helper"], ["--upload-pack"]), + (["conf"], ["--config"]), + (["--out=/tmp/output"], ["--output"]), + ] + + for options, unsafe_options in cases: + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=options, unsafe_options=unsafe_options) + + def test_check_unsafe_options_does_not_treat_short_options_as_abbreviations(self): + unsafe_options = ["--upload-pack"] + + Git.check_unsafe_options(options=["-u", "u", "-upl"], unsafe_options=unsafe_options) + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=["--u"], unsafe_options=unsafe_options) + + def test_check_unsafe_options_does_not_treat_option_values_as_abbreviations(self): + Git.check_unsafe_options(options=["--branch", "conf"], unsafe_options=["--config"]) + + def test_check_unsafe_options_rejects_joined_unsafe_short_options(self): + cases = [ + (["-utouch /tmp/pwn"], ["-u"]), + (["-futouch /tmp/pwn"], ["-u"]), + (["-qutouch${IFS}/tmp/pwn"], ["-u"]), + (["-cprotocol.ext.allow=always"], ["-c"]), + (["-vcprotocol.ext.allow=always"], ["-c"]), + ] + + for options, unsafe_options in cases: + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=options, unsafe_options=unsafe_options) + + def test_check_unsafe_options_allows_attached_safe_short_option_values(self): + unsafe_options = ["--upload-pack", "-u", "--config", "-c"] + + Git.check_unsafe_options(options=["-oupstream", "-bcurrent"], unsafe_options=unsafe_options) + + def test_option_candidates_ignore_untransformed_kwargs(self): + options = Git._option_candidates( + kwargs={ + "output": None, + "upload_pack": False, + "exec": [], + "config": [None, False], + "max_count": 1, + } + ) + + self.assertEqual(options, ["--max-count"]) + + def test_option_candidates_include_falsey_non_boolean_values(self): + kwargs = {"pathspec_from_file": 0} + candidates = Git._option_candidates(kwargs=kwargs) + + self.assertEqual(candidates, ["--pathspec-from-file"]) + self.assertEqual(self.git.transform_kwargs(**kwargs), ["--pathspec-from-file=0"]) + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options( + options=candidates, + unsafe_options=Git.unsafe_git_pathspec_from_file_options, + ) + + def test_option_candidates_include_split_single_char_option_values(self): + cases = [ + ({"n": "--upload-pack=helper"}, ["-n", "--upload-pack=helper"], ["--upload-pack"]), + ({"g": ("safe", "--out=target")}, ["-g", "--out=target"], ["--output"]), + ] + + for kwargs, candidates, unsafe_options in cases: + self.assertEqual(Git._option_candidates(kwargs=kwargs), candidates) + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=candidates, unsafe_options=unsafe_options) + + self.assertEqual(self.git.transform_kwargs(n="--upload-pack=helper"), ["-n", "--upload-pack=helper"]) + + unsplit_kwargs = {"n": "--upload-pack=helper", "split_single_char_options": False} + self.assertEqual(self.git.transform_kwargs(**unsplit_kwargs), ["-n--upload-pack=helper"]) + self.assertEqual(Git._option_candidates(kwargs=unsplit_kwargs), ["-n--upload-pack=helper"]) + + def test_option_candidates_include_joined_single_char_option_values(self): + kwargs = {"n": "uhelper", "split_single_char_options": False} + candidates = Git._option_candidates(kwargs=kwargs) + + self.assertEqual(candidates, ["-nuhelper"]) + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=candidates, unsafe_options=["-u"]) + _shell_cases = ( # value_in_call, value_from_class, expected_popen_arg (None, False, False), @@ -133,7 +283,7 @@ def test_it_uses_shell_or_not_as_specified(self, case): def test_it_logs_if_it_uses_a_shell(self, case): """``shell=`` in the log message agrees with what is passed to `Popen`.""" value_in_call, value_from_class = case - with self.assertLogs(cmd.log, level=logging.DEBUG) as log_watcher: + with self.assertLogs(cmd.__name__, level=logging.DEBUG) as log_watcher: mock_safer_popen = self._do_shell_combo(value_in_call, value_from_class) self._assert_logged_for_popen(log_watcher, "shell", mock_safer_popen.call_args.kwargs["shell"]) @@ -143,13 +293,55 @@ def test_it_logs_if_it_uses_a_shell(self, case): ) def test_it_logs_istream_summary_for_stdin(self, case): expected_summary, istream_argument = case - with self.assertLogs(cmd.log, level=logging.DEBUG) as log_watcher: + with self.assertLogs(cmd.__name__, level=logging.DEBUG) as log_watcher: self.git.execute(["git", "version"], istream=istream_argument) self._assert_logged_for_popen(log_watcher, "stdin", expected_summary) def test_it_executes_git_and_returns_result(self): self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") + def test_it_output_stream_with_stdout_is_false(self): + temp_stream = io.BytesIO() + self.git.execute( + ["git", "version"], + output_stream=temp_stream, + with_stdout=False, + ) + self.assertEqual(temp_stream.tell(), 0) + + @skipUnless(sys.platform != "win32", "kill_after_timeout is not supported on Windows") + def test_it_honors_kill_after_timeout_with_output_stream(self): + output_stream = io.BytesIO() + command = [ + sys.executable, + "-c", + "import sys, time; sys.stdout.write('started\\n'); sys.stdout.flush(); sys.stdout.close(); time.sleep(60)", + ] + + started = time.monotonic() + status, _, stderr = self.git.execute( + command, + output_stream=output_stream, + kill_after_timeout=1, + with_exceptions=False, + with_extended_output=True, + ) + + self.assertLess(time.monotonic() - started, 5) + self.assertNotEqual(status, 0) + self.assertEqual(output_stream.getvalue(), b"started\n") + self.assertIn("Timeout: the command", stderr) + + def test_it_executes_git_without_stdout_redirect(self): + returncode, stdout, stderr = self.git.execute( + ["git", "version"], + with_extended_output=True, + with_stdout=False, + ) + self.assertEqual(returncode, 0) + self.assertIsNone(stdout) + self.assertIsNotNone(stderr) + @ddt.data( # chdir_to_repo, shell, command, use_shell_impostor (False, False, ["git", "version"], False), @@ -165,7 +357,7 @@ def test_it_executes_git_not_from_cwd(self, rw_dir, case): repo = Repo.init(rw_dir) - if os.name == "nt": + if sys.platform == "win32": # Copy an actual binary executable that is not git. (On Windows, running # "hostname" only displays the hostname, it never tries to change it.) other_exe_path = Path(os.environ["SystemRoot"], "system32", "hostname.exe") @@ -178,7 +370,7 @@ def test_it_executes_git_not_from_cwd(self, rw_dir, case): os.chmod(impostor_path, 0o755) if use_shell_impostor: - shell_name = "cmd.exe" if os.name == "nt" else "sh" + shell_name = "cmd.exe" if sys.platform == "win32" else "sh" shutil.copy(impostor_path, Path(rw_dir, shell_name)) with contextlib.ExitStack() as stack: @@ -195,7 +387,7 @@ def test_it_executes_git_not_from_cwd(self, rw_dir, case): self.assertRegex(output, r"^git version\b") @skipUnless( - os.name == "nt", + sys.platform == "win32", "The regression only affected Windows, and this test logic is OS-specific.", ) def test_it_avoids_upcasing_unrelated_environment_variable_names(self): @@ -203,21 +395,27 @@ def test_it_avoids_upcasing_unrelated_environment_variable_names(self): if old_name == old_name.upper(): raise RuntimeError("test bug or strange locale: old_name invariant under upcasing") - # Step 1: Set the environment variable in this parent process. Because os.putenv is a thin - # wrapper around a system API, os.environ never sees the variable in this parent - # process, so the name is not upcased even on Windows. + # Step 1 + # + # Set the environment variable in this parent process. Because os.putenv is a + # thin wrapper around a system API, os.environ never sees the variable in this + # parent process, so the name is not upcased even on Windows. os.putenv(old_name, "1") - # Step 2: Create the child process that inherits the environment variable. The child uses - # GitPython, and we are testing that it passes the variable with the exact original - # name to its own child process (the grandchild). + # Step 2 + # + # Create the child process that inherits the environment variable. The child + # uses GitPython, and we are testing that it passes the variable with the exact + # original name to its own child process (the grandchild). cmdline = [ sys.executable, fixture_path("env_case.py"), # Contains steps 3 and 4. self.rorepo.working_dir, old_name, ] - pair_text = subprocess.check_output(cmdline, shell=False, text=True) # Run steps 3 and 4. + + # Run steps 3 and 4. + pair_text = subprocess.check_output(cmdline, shell=False, text=True) new_name = pair_text.split("=")[0] self.assertEqual(new_name, old_name) @@ -237,13 +435,27 @@ def test_it_ignores_false_kwargs(self, git): self.assertTrue("pass_this_kwarg" not in git.call_args[1]) def test_it_raises_proper_exception_with_output_stream(self): - tmp_file = TemporaryFile() - self.assertRaises( - GitCommandError, - self.git.checkout, - "non-existent-branch", - output_stream=tmp_file, - ) + with tempfile.TemporaryFile() as tmp_file: + with self.assertRaises(GitCommandError): + self.git.checkout("non-existent-branch", output_stream=tmp_file) + + def test_it_redacts_authorization_extra_header_from_error(self): + token = "fake-token-1234" + command = [ + "git", + "-c", + "http.extraHeader=Authorization: Bearer %s" % token, + "rev-parse", + "--verify", + "refs/does-not-exist", + ] + + with self.assertRaises(GitCommandError) as context: + self.git.execute(command) + + message = str(context.exception) + self.assertNotIn(token, message) + self.assertIn("http.extraHeader=Authorization: *****", message) def test_it_accepts_environment_variables(self): filename = fixture_path("ls_tree_empty") @@ -291,29 +503,348 @@ def test_persistent_cat_file_command(self): self.assertEqual(typename, typename_two) self.assertEqual(size, size_two) - def test_version(self): + def test_version_info(self): + """The version_info attribute is a tuple of up to four ints.""" v = self.git.version_info self.assertIsInstance(v, tuple) + self.assertLessEqual(len(v), 4) + for n in v: + self.assertIsInstance(n, int) + + def test_version_info_pickleable(self): + """The version_info attribute is usable on unpickled Git instances.""" + deserialized = pickle.loads(pickle.dumps(self.git)) + v = deserialized.version_info + self.assertIsInstance(v, tuple) + self.assertLessEqual(len(v), 4) for n in v: self.assertIsInstance(n, int) - # END verify number types + + @ddt.data( + (("123", "456", "789"), (123, 456, 789)), + (("12", "34", "56", "78"), (12, 34, 56, 78)), + (("12", "34", "56", "78", "90"), (12, 34, 56, 78)), + (("1", "2", "a", "3"), (1, 2)), + (("1", "-2", "3"), (1,)), + (("1", "2a", "3"), (1,)), # Subject to change. + ) + def test_version_info_is_leading_numbers(self, case): + fake_fields, expected_version_info = case + with _rollback_refresh(): + with _fake_git(*fake_fields) as path: + refresh(path) + new_git = Git() + self.assertEqual(new_git.version_info, expected_version_info) + + def test_git_exc_name_is_git(self): + self.assertEqual(self.git.git_exec_name, "git") def test_cmd_override(self): - with mock.patch.object( - type(self.git), - "GIT_PYTHON_GIT_EXECUTABLE", - osp.join("some", "path", "which", "doesn't", "exist", "gitbinary"), - ): - self.assertRaises(GitCommandNotFound, self.git.version) - - def test_refresh(self): - # Test a bad git path refresh. - self.assertRaises(GitCommandNotFound, refresh, "yada") - - # Test a good path refresh. - which_cmd = "where" if os.name == "nt" else "command -v" - path = os.popen("{0} git".format(which_cmd)).read().strip().split("\n")[0] - refresh(path) + """Directly set bad GIT_PYTHON_GIT_EXECUTABLE causes git operations to raise.""" + bad_path = osp.join("some", "path", "which", "doesn't", "exist", "gitbinary") + with mock.patch.object(Git, "GIT_PYTHON_GIT_EXECUTABLE", bad_path): + with self.assertRaises(GitCommandNotFound) as ctx: + self.git.version() + self.assertEqual(ctx.exception.command, [bad_path, "version"]) + + @ddt.data(("0",), ("q",), ("quiet",), ("s",), ("silence",), ("silent",), ("n",), ("none",)) + def test_initial_refresh_from_bad_git_path_env_quiet(self, case): + """In "q" mode, bad initial path sets "git" and is quiet.""" + (mode,) = case + set_vars = { + "GIT_PYTHON_GIT_EXECUTABLE": str(Path("yada").absolute()), # Any bad path. + "GIT_PYTHON_REFRESH": mode, + } + with _rollback_refresh(): + Git.GIT_PYTHON_GIT_EXECUTABLE = None # Simulate startup. + + with mock.patch.dict(os.environ, set_vars): + refresh() + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, "git") + + @ddt.data(("1",), ("w",), ("warn",), ("warning",), ("l",), ("log",)) + def test_initial_refresh_from_bad_git_path_env_warn(self, case): + """In "w" mode, bad initial path sets "git" and warns, by logging.""" + (mode,) = case + env_vars = { + "GIT_PYTHON_GIT_EXECUTABLE": str(Path("yada").absolute()), # Any bad path. + "GIT_PYTHON_REFRESH": mode, + } + with _rollback_refresh(): + Git.GIT_PYTHON_GIT_EXECUTABLE = None # Simulate startup. + + with mock.patch.dict(os.environ, env_vars): + with self.assertLogs(cmd.__name__, logging.CRITICAL) as ctx: + refresh() + self.assertEqual(len(ctx.records), 1) + message = ctx.records[0].getMessage() + self.assertRegex(message, r"\ABad git executable.\n") + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, "git") + + @ddt.data(("2",), ("r",), ("raise",), ("e",), ("error",)) + def test_initial_refresh_from_bad_git_path_env_error(self, case): + """In "e" mode, bad initial path raises an exception.""" + (mode,) = case + env_vars = { + "GIT_PYTHON_GIT_EXECUTABLE": str(Path("yada").absolute()), # Any bad path. + "GIT_PYTHON_REFRESH": mode, + } + with _rollback_refresh(): + Git.GIT_PYTHON_GIT_EXECUTABLE = None # Simulate startup. + + with mock.patch.dict(os.environ, env_vars): + with self.assertRaisesRegex(ImportError, r"\ABad git executable.\n"): + refresh() + + def test_initial_refresh_from_good_absolute_git_path_env(self): + """Good initial absolute path from environment is set.""" + absolute_path = shutil.which("git") + + with _rollback_refresh(): + Git.GIT_PYTHON_GIT_EXECUTABLE = None # Simulate startup. + + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": absolute_path}): + refresh() + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, absolute_path) + + def test_initial_refresh_from_good_relative_git_path_env(self): + """Good initial relative path from environment is kept relative and set.""" + with _rollback_refresh(): + # Set the fallback to a string that wouldn't work and isn't "git", so we are + # more likely to detect if "git" is not set from the environment variable. + with mock.patch.object(Git, "git_exec_name", ""): + Git.GIT_PYTHON_GIT_EXECUTABLE = None # Simulate startup. + + # Now observe if setting the environment variable to "git" takes effect. + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": "git"}): + refresh() + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, "git") + + def test_refresh_from_bad_absolute_git_path_env(self): + """Bad absolute path from environment is reported and not set.""" + absolute_path = str(Path("yada").absolute()) + expected_pattern = rf"\n[ \t]*cmdline: {re.escape(absolute_path)}\Z" + + with _rollback_refresh() as old_git_executable: + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": absolute_path}): + with self.assertRaisesRegex(GitCommandNotFound, expected_pattern): + refresh() + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, old_git_executable) + + def test_refresh_from_bad_relative_git_path_env(self): + """Bad relative path from environment is kept relative and reported, not set.""" + # Relative paths are not resolved when refresh() is called with no arguments, so + # use a string that's very unlikely to be a command name found in a path lookup. + relative_path = "yada-e47e70c6-acbf-40f8-ad65-13af93c2195b" + expected_pattern = rf"\n[ \t]*cmdline: {re.escape(relative_path)}\Z" + + with _rollback_refresh() as old_git_executable: + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": relative_path}): + with self.assertRaisesRegex(GitCommandNotFound, expected_pattern): + refresh() + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, old_git_executable) + + def test_refresh_from_good_absolute_git_path_env(self): + """Good absolute path from environment is set.""" + absolute_path = shutil.which("git") + + with _rollback_refresh(): + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": absolute_path}): + refresh() + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, absolute_path) + + def test_refresh_from_good_relative_git_path_env(self): + """Good relative path from environment is kept relative and set.""" + with _rollback_refresh(): + # Set as the executable name a string that wouldn't work and isn't "git". + Git.GIT_PYTHON_GIT_EXECUTABLE = "" + + # Now observe if setting the environment variable to "git" takes effect. + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": "git"}): + refresh() + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, "git") + + def test_refresh_with_bad_absolute_git_path_arg(self): + """Bad absolute path arg is reported and not set.""" + absolute_path = str(Path("yada").absolute()) + expected_pattern = rf"\n[ \t]*cmdline: {re.escape(absolute_path)}\Z" + + with _rollback_refresh() as old_git_executable: + with self.assertRaisesRegex(GitCommandNotFound, expected_pattern): + refresh(absolute_path) + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, old_git_executable) + + def test_refresh_with_bad_relative_git_path_arg(self): + """Bad relative path arg is resolved to absolute path and reported, not set.""" + absolute_path = str(Path("yada").absolute()) + expected_pattern = rf"\n[ \t]*cmdline: {re.escape(absolute_path)}\Z" + + with _rollback_refresh() as old_git_executable: + with self.assertRaisesRegex(GitCommandNotFound, expected_pattern): + refresh("yada") + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, old_git_executable) + + def test_refresh_with_good_absolute_git_path_arg(self): + """Good absolute path arg is set.""" + absolute_path = shutil.which("git") + + with _rollback_refresh(): + refresh(absolute_path) + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, absolute_path) + + def test_refresh_with_good_relative_git_path_arg(self): + """Good relative path arg is resolved to absolute path and set.""" + absolute_path = shutil.which("git") + dirname, basename = osp.split(absolute_path) + + with cwd(dirname): + with _rollback_refresh(): + refresh(basename) + self.assertEqual(self.git.GIT_PYTHON_GIT_EXECUTABLE, absolute_path) + + def test_version_info_is_cached(self): + fake_version_info = (123, 456, 789) + with _rollback_refresh(): + with _fake_git(*fake_version_info) as path: + new_git = Git() # Not cached yet. + refresh(path) + self.assertEqual(new_git.version_info, fake_version_info) + os.remove(path) # Arrange that a second subprocess call would fail. + self.assertEqual(new_git.version_info, fake_version_info) + + def test_version_info_cache_is_per_instance(self): + with _rollback_refresh(): + with _fake_git(123, 456, 789) as path: + git1 = Git() + git2 = Git() + refresh(path) + git1.version_info + os.remove(path) # Arrange that the second subprocess call will fail. + with self.assertRaises(GitCommandNotFound): + git2.version_info + git1.version_info + + def test_version_info_cache_is_not_pickled(self): + with _rollback_refresh(): + with _fake_git(123, 456, 789) as path: + git1 = Git() + refresh(path) + git1.version_info + git2 = pickle.loads(pickle.dumps(git1)) + os.remove(path) # Arrange that the second subprocess call will fail. + with self.assertRaises(GitCommandNotFound): + git2.version_info + git1.version_info + + def test_successful_refresh_with_arg_invalidates_cached_version_info(self): + with _rollback_refresh(): + with _fake_git(11, 111, 1) as path1: + with _fake_git(22, 222, 2) as path2: + new_git = Git() + refresh(path1) + new_git.version_info + refresh(path2) + self.assertEqual(new_git.version_info, (22, 222, 2)) + + def test_failed_refresh_with_arg_does_not_invalidate_cached_version_info(self): + with _rollback_refresh(): + with _fake_git(11, 111, 1) as path1: + with _fake_git(22, 222, 2) as path2: + new_git = Git() + refresh(path1) + new_git.version_info + os.remove(path1) # Arrange that a repeat call for path1 would fail. + os.remove(path2) # Arrange that the new call for path2 will fail. + with self.assertRaises(GitCommandNotFound): + refresh(path2) + self.assertEqual(new_git.version_info, (11, 111, 1)) + + def test_successful_refresh_with_same_arg_invalidates_cached_version_info(self): + """Changing git at the same path and refreshing affects version_info.""" + with _rollback_refresh(): + with _fake_git(11, 111, 1) as path1: + with _fake_git(22, 222, 2) as path2: + new_git = Git() + refresh(path1) + new_git.version_info + shutil.copy(path2, path1) + refresh(path1) # The fake git at path1 has a different version now. + self.assertEqual(new_git.version_info, (22, 222, 2)) + + def test_successful_refresh_with_env_invalidates_cached_version_info(self): + with contextlib.ExitStack() as stack: + stack.enter_context(_rollback_refresh()) + path1 = stack.enter_context(_fake_git(11, 111, 1)) + path2 = stack.enter_context(_fake_git(22, 222, 2)) + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": path1}): + new_git = Git() + refresh() + new_git.version_info + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": path2}): + refresh() + self.assertEqual(new_git.version_info, (22, 222, 2)) + + def test_failed_refresh_with_env_does_not_invalidate_cached_version_info(self): + with contextlib.ExitStack() as stack: + stack.enter_context(_rollback_refresh()) + path1 = stack.enter_context(_fake_git(11, 111, 1)) + path2 = stack.enter_context(_fake_git(22, 222, 2)) + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": path1}): + new_git = Git() + refresh() + new_git.version_info + os.remove(path1) # Arrange that a repeat call for path1 would fail. + os.remove(path2) # Arrange that the new call for path2 will fail. + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": path2}): + with self.assertRaises(GitCommandNotFound): + refresh(path2) + self.assertEqual(new_git.version_info, (11, 111, 1)) + + def test_successful_refresh_with_same_env_invalidates_cached_version_info(self): + """Changing git at the same path/command and refreshing affects version_info.""" + with contextlib.ExitStack() as stack: + stack.enter_context(_rollback_refresh()) + path1 = stack.enter_context(_fake_git(11, 111, 1)) + path2 = stack.enter_context(_fake_git(22, 222, 2)) + with mock.patch.dict(os.environ, {"GIT_PYTHON_GIT_EXECUTABLE": path1}): + new_git = Git() + refresh() + new_git.version_info + shutil.copy(path2, path1) + refresh() # The fake git at path1 has a different version now. + self.assertEqual(new_git.version_info, (22, 222, 2)) + + def test_successful_default_refresh_invalidates_cached_version_info(self): + """Refreshing updates version after a filesystem change adds a git command.""" + # The key assertion here is the last. The others mainly verify the test itself. + with contextlib.ExitStack() as stack: + stack.enter_context(_rollback_refresh()) + + path1 = Path(stack.enter_context(_fake_git(11, 111, 1))) + path2 = Path(stack.enter_context(_fake_git(22, 222, 2))) + + new_path_var = f"{path1.parent}{os.pathsep}{path2.parent}" + stack.enter_context(mock.patch.dict(os.environ, {"PATH": new_path_var})) + stack.enter_context(_patch_out_env("GIT_PYTHON_GIT_EXECUTABLE")) + + if sys.platform == "win32": + # On Windows, use a shell so "git" finds "git.cmd". The correct and safe + # ways to do this straightforwardly are to set GIT_PYTHON_GIT_EXECUTABLE + # to git.cmd in the environment, or call git.refresh with the command's + # full path. See the Git.USE_SHELL docstring for deprecation details. + # But this tests a "default" scenario where neither is done. The + # approach used here, setting USE_SHELL to True so PATHEXT is honored, + # should not be used in production code (nor even in most test cases). + stack.enter_context(mock.patch.object(Git, "USE_SHELL", True)) + + new_git = Git() + _rename_with_stem(path2, "git") # "Install" git, "late" in the PATH. + refresh() + self.assertEqual(new_git.version_info, (22, 222, 2), 'before "downgrade"') + _rename_with_stem(path1, "git") # "Install" another, higher priority. + self.assertEqual(new_git.version_info, (22, 222, 2), "stale version") + refresh() + self.assertEqual(new_git.version_info, (11, 111, 1), "fresh version") def test_options_are_passed_to_git(self): # This works because any command after git --version is ignored. @@ -375,7 +906,7 @@ def test_environment(self, rw_dir): path = osp.join(rw_dir, "failing-script.sh") with open(path, "wt") as stream: - stream.write("#!/usr/bin/env sh\n" "echo FOO\n") + stream.write("#!/usr/bin/env sh\necho FOO\n") os.chmod(path, 0o777) rw_repo = Repo.init(osp.join(rw_dir, "repo")) @@ -390,17 +921,18 @@ def test_environment(self, rw_dir): def test_handle_process_output(self): from git.cmd import handle_process_output, safer_popen - line_count = 5002 - count = [None, 0, 0] + expected_line_count = 5002 + actual_lines = [None, [], []] - def counter_stdout(line): - count[1] += 1 + def stdout_handler(line): + actual_lines[1].append(line) - def counter_stderr(line): - count[2] += 1 + def stderr_handler(line): + actual_lines[2].append(line) cmdline = [ sys.executable, + "-S", # Keep any `CoverageWarning` messages out of the subprocess stderr. fixture_path("cat_file.py"), str(fixture_path("issue-301_stderr")), ] @@ -412,10 +944,10 @@ def counter_stderr(line): shell=False, ) - handle_process_output(proc, counter_stdout, counter_stderr, finalize_process) + handle_process_output(proc, stdout_handler, stderr_handler, finalize_process) - self.assertEqual(count[1], line_count) - self.assertEqual(count[2], line_count) + self.assertEqual(len(actual_lines[1]), expected_line_count, repr(actual_lines[1])) + self.assertEqual(len(actual_lines[2]), expected_line_count, repr(actual_lines[2])) def test_execute_kwargs_set_agrees_with_method(self): parameter_names = inspect.signature(cmd.Git.execute).parameters.keys() diff --git a/test/test_imports.py b/test/test_imports.py new file mode 100644 index 000000000..8e70c6689 --- /dev/null +++ b/test/test_imports.py @@ -0,0 +1,32 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +import sys + +import git + + +def test_git_util_attribute_is_git_index_util(): + """The top-level module's ``util`` attribute is really :mod:`git.index.util`. + + Although this situation is unintuitive and not a design goal, this has historically + been the case, and it should not be changed without considering the effect on + backward compatibility. In practice, it cannot be changed at least until the next + major version of GitPython. This test checks that it is not accidentally changed, + which could happen when refactoring imports. + """ + assert git.util is git.index.util + + +def test_git_index_util_attribute_is_git_index_util(): + """Nothing unusual is happening with git.index.util itself.""" + assert git.index.util is sys.modules["git.index.util"] + + +def test_separate_git_util_module_exists(): + """The real git.util and git.index.util modules really are separate. + + The real git.util module can be accessed to import a name ``...` by writing + ``from git.util import ...``, and the module object can be accessed in sys.modules. + """ + assert sys.modules["git.util"] is not sys.modules["git.index.util"] diff --git a/test/test_index.py b/test/test_index.py index 8a64e2293..469a9dc17 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -4,6 +4,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib +from dataclasses import dataclass from io import BytesIO import logging import os @@ -13,46 +14,37 @@ import shutil from stat import S_ISLNK, ST_MODE import subprocess +import sys import tempfile +from unittest import mock + +from gitdb.base import IStream import ddt import pytest -from sumtypes import constructor, sumtype - -from git import ( - BlobFilter, - Diff, - Git, - IndexFile, - Object, - Repo, - Tree, -) + +from git import BlobFilter, Diff, Git, IndexFile, NULL_TREE, Object, Repo, Tree +from git.diff import NULL_TREE_SHA from git.exc import ( CheckoutError, GitCommandError, HookExecutionError, InvalidGitRepositoryError, UnmergedEntriesError, + UnsafeOptionError, ) -from git.index.fun import hook_path, run_commit_hook +from git.index.fun import _git_for_windows_bash, _which_from_path, hook_path, run_commit_hook from git.index.typ import BaseIndexEntry, IndexEntry from git.index.util import TemporaryFileSwap from git.objects import Blob from git.util import Actor, cwd, hex_to_bin, rmtree -from gitdb.base import IStream -from test.lib import ( - TestBase, - VirtualEnvironment, - fixture, - fixture_path, - with_rw_directory, - with_rw_repo, -) + +from test.lib import TestBase, VirtualEnvironment, fixture, fixture_path, with_rw_directory, with_rw_repo, PathLikeMock +from test.lib.helper import symlinks_supported, xfail_if_raises HOOKS_SHEBANG = "#!/usr/bin/env sh\n" -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) def _get_windows_ansi_encoding(): @@ -66,34 +58,48 @@ def _get_windows_ansi_encoding(): return f"cp{value}" -@sumtype class WinBashStatus: - """Status of bash.exe for native Windows. Affects which commit hook tests can pass. + """Namespace of native-Windows bash.exe statuses. Affects what hook tests can pass. Call check() to check the status. (CheckError and WinError should not typically be used to trigger skip or xfail, because they represent unexpected situations.) """ - Inapplicable = constructor() - """This system is not native Windows: either not Windows at all, or Cygwin.""" + @dataclass + class Inapplicable: + """This system is not native Windows: either not Windows at all, or Cygwin.""" + + @dataclass + class Absent: + """No command for bash.exe is found on the system.""" + + @dataclass + class Native: + """Running bash.exe operates outside any WSL distribution (as with Git Bash).""" - Absent = constructor() - """No command for bash.exe is found on the system.""" + @dataclass + class Wsl: + """Running bash.exe calls bash in a WSL distribution.""" - Native = constructor() - """Running bash.exe operates outside any WSL distribution (as with Git Bash).""" + @dataclass + class WslNoDistro: + """Running bash.exe tries to run bash on a WSL distribution, but none exists.""" - Wsl = constructor() - """Running bash.exe calls bash in a WSL distribution.""" + process: "subprocess.CompletedProcess[bytes]" + message: str - WslNoDistro = constructor("process", "message") - """Running bash.exe tries to run bash on a WSL distribution, but none exists.""" + @dataclass + class CheckError: + """Running bash.exe fails in an unexpected error or gives unexpected output.""" - CheckError = constructor("process", "message") - """Running bash.exe fails in an unexpected error or gives unexpected output.""" + process: "subprocess.CompletedProcess[bytes]" + message: str - WinError = constructor("exception") - """bash.exe may exist but can't run. CreateProcessW fails unexpectedly.""" + @dataclass + class WinError: + """bash.exe may exist but can't run. CreateProcessW fails unexpectedly.""" + + exception: OSError @classmethod def check(cls): @@ -110,7 +116,7 @@ def check(cls): in System32; Popen finds it even if a shell would run another one, as on CI. (Without WSL, System32 may still have bash.exe; users sometimes put it there.) """ - if os.name != "nt": + if sys.platform != "win32": return cls.Inapplicable() try: @@ -130,13 +136,13 @@ def check(cls): if process.returncode == 1 and re.search(r"\bhttps://aka.ms/wslstore\b", text): return cls.WslNoDistro(process, text) if process.returncode != 0: - log.error("Error running bash.exe to check WSL status: %s", text) + _logger.error("Error running bash.exe to check WSL status: %s", text) return cls.CheckError(process, text) if text == "0": return cls.Wsl() if text == "1": return cls.Native() - log.error("Strange output checking WSL status: %s", text) + _logger.error("Strange output checking WSL status: %s", text) return cls.CheckError(process, text) @staticmethod @@ -160,7 +166,7 @@ def _decode(stdout): except UnicodeDecodeError: pass except LookupError as error: - log.warning("%s", str(error)) # Message already says "Unknown encoding:". + _logger.warning(str(error)) # Message already says "Unknown encoding:". # Assume UTF-8. If invalid, substitute Unicode replacement characters. return stdout.decode("utf-8", errors="replace") @@ -184,6 +190,41 @@ def _make_hook(git_dir, name, content, make_exec=True): @ddt.ddt class TestIndex(TestBase): + @with_rw_repo("HEAD") + def test_checkout_rejects_unsafe_prefix(self, rw_repo): + with tempfile.TemporaryDirectory() as target: + with self.assertRaises(UnsafeOptionError): + rw_repo.index.checkout(prefix=f"{target}/") + + rw_repo.index.checkout(prefix=f"{target}/", allow_unsafe_options=True) + self.assertTrue(osp.isfile(osp.join(target, "CHANGES"))) + + @with_rw_repo("HEAD") + def test_remove_rejects_pathspec_from_file(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + pathspecs = Path(tdir) / "pathspecs" + pathspecs.write_bytes(b"unmatched-path-one\nunmatched-path-two") + for option_name in ("pathspec_from_file", "pathspec_from"): + with self.assertRaises(UnsafeOptionError): + rw_repo.index.remove( + [], + pathspec_file_nul=True, + **{option_name: str(pathspecs)}, + ) + + @with_rw_repo("HEAD") + def test_remove_allows_explicit_pathspec_from_file(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + pathspecs = Path(tdir) / "pathspecs" + pathspecs.write_bytes(b"CHANGES\0") + removed = rw_repo.index.remove( + [], + pathspec_from_file=str(pathspecs), + pathspec_file_nul=True, + allow_unsafe_options=True, + ) + assert "CHANGES" in removed + def __init__(self, *args): super().__init__(*args) self._reset_progress() @@ -205,8 +246,7 @@ def _fprogress(self, path, done, item): self._fprogress_map[path] = curval + 1 def _fprogress_add(self, path, done, item): - """Called as progress func - we keep track of the proper - call order""" + """Called as progress func - we keep track of the proper call order.""" assert item is not None self._fprogress(path, done, item) @@ -300,6 +340,22 @@ def add_bad_blob(): except Exception as ex: assert "index.lock' could not be obtained" not in str(ex) + @with_rw_repo("0.1.6") + def test_read_tree_methods_reject_index_output(self, rw_repo): + output_path = (Path(rw_repo.working_tree_dir) / "alternate-index").as_posix() + unsafe_option = f"--index-output={output_path}" + + with pytest.raises(UnsafeOptionError): + IndexFile.from_tree(rw_repo, unsafe_option) + with pytest.raises(UnsafeOptionError): + IndexFile.from_tree(rw_repo, "HEAD", index_output=output_path) + with pytest.raises(UnsafeOptionError): + rw_repo.index.reset(unsafe_option) + with pytest.raises(UnsafeOptionError): + rw_repo.index.merge_tree(unsafe_option) + with pytest.raises(UnsafeOptionError): + rw_repo.index.merge_tree("HEAD", base=unsafe_option) + @with_rw_repo("0.1.6") def test_index_file_from_tree(self, rw_repo): common_ancestor_sha = "5117c9c8a4d3af19a9958677e45cda9269de1541" @@ -321,7 +377,10 @@ def test_index_file_from_tree(self, rw_repo): assert len([e for e in three_way_index.entries.values() if e.stage != 0]) # ITERATE BLOBS - merge_required = lambda t: t[0] != 0 + + def merge_required(t): + return t[0] != 0 + merge_blobs = list(three_way_index.iter_blobs(merge_required)) assert merge_blobs assert merge_blobs[0][0] in (1, 2, 3) @@ -371,17 +430,17 @@ def test_index_merge_tree(self, rw_repo): # FAKE MERGE ############# - # Add a change with a NULL sha that should conflict with next_commit. We - # pretend there was a change, but we do not even bother adding a proper - # sha for it (which makes things faster of course). + # Add a change with a NULL sha that should conflict with next_commit. We pretend + # there was a change, but we do not even bother adding a proper sha for it + # (which makes things faster of course). manifest_fake_entry = BaseIndexEntry((manifest_entry[0], b"\0" * 20, 0, manifest_entry[3])) # Try write flag. self._assert_entries(rw_repo.index.add([manifest_fake_entry], write=False)) - # Add actually resolves the null-hex-sha for us as a feature, but we can - # edit the index manually. + # Add actually resolves the null-hex-sha for us as a feature, but we can edit + # the index manually. assert rw_repo.index.entries[manifest_key].binsha != Object.NULL_BIN_SHA - # We must operate on the same index for this! It's a bit problematic as - # it might confuse people. + # We must operate on the same index for this! It's a bit problematic as it might + # confuse people. index = rw_repo.index index.entries[manifest_key] = IndexEntry.from_base(manifest_fake_entry) index.write() @@ -390,19 +449,20 @@ def test_index_merge_tree(self, rw_repo): # Write an unchanged index (just for the fun of it). rw_repo.index.write() - # A three way merge would result in a conflict and fails as the command will - # not overwrite any entries in our index and hence leave them unmerged. This is + # A three way merge would result in a conflict and fails as the command will not + # overwrite any entries in our index and hence leave them unmerged. This is # mainly a protection feature as the current index is not yet in a tree. self.assertRaises(GitCommandError, index.merge_tree, next_commit, base=parent_commit) - # The only way to get the merged entries is to safe the current index away into a tree, - # which is like a temporary commit for us. This fails as well as the NULL sha does not - # have a corresponding object. + # The only way to get the merged entries is to safe the current index away into + # a tree, which is like a temporary commit for us. This fails as well as the + # NULL sha does not have a corresponding object. # NOTE: missing_ok is not a kwarg anymore, missing_ok is always true. # self.assertRaises(GitCommandError, index.write_tree) - # If missing objects are okay, this would work though (they are always okay now). - # As we can't read back the tree with NULL_SHA, we rather set it to something else. + # If missing objects are okay, this would work though (they are always okay + # now). As we can't read back the tree with NULL_SHA, we rather set it to + # something else. index.entries[manifest_key] = IndexEntry(manifest_entry[:1] + (hex_to_bin("f" * 40),) + manifest_entry[2:]) tree = index.write_tree() @@ -414,7 +474,7 @@ def test_index_merge_tree(self, rw_repo): @with_rw_repo("0.1.6") def test_index_file_diffing(self, rw_repo): - # Default Index instance points to our index. + # Default IndexFile instance points to our index. index = IndexFile(rw_repo) assert index.path is not None assert len(index.entries) @@ -425,8 +485,8 @@ def test_index_file_diffing(self, rw_repo): # Could sha it, or check stats. # Test diff. - # Resetting the head will leave the index in a different state, and the - # diff will yield a few changes. + # Resetting the head will leave the index in a different state, and the diff + # will yield a few changes. cur_head_commit = rw_repo.head.reference.commit rw_repo.head.reset("HEAD~6", index=True, working_tree=False) @@ -514,7 +574,7 @@ def test_index_file_diffing(self, rw_repo): index.checkout(test_file) except CheckoutError as e: # Detailed exceptions are only possible in older git versions. - if rw_repo.git._version_info < (2, 29): + if rw_repo.git.version_info < (2, 29): self.assertEqual(len(e.failed_files), 1) self.assertEqual(e.failed_files[0], osp.basename(test_file)) self.assertEqual(len(e.failed_files), len(e.failed_reasons)) @@ -535,6 +595,39 @@ def test_index_file_diffing(self, rw_repo): rval = index.checkout("lib") assert len(list(rval)) > 1 + @with_rw_directory + def test_index_file_diff_null_tree_with_initial_index(self, rw_dir): + repo = Repo.init(rw_dir) + filename = ".gitkeep" + file_path = osp.join(repo.working_tree_dir, filename) + with open(file_path, "w") as fp: + fp.write("# Initial file\n") + + index = repo.index + index.add([filename]) + index.write() + + index = IndexFile(repo) + self.assertEqual(len(index.diff(None)), 0) + + diff = index.diff(NULL_TREE) + self.assertEqual(len(diff), 1) + self.assertEqual(diff[0].change_type, "A") + assert diff[0].new_file + self.assertEqual(diff[0].b_path, filename) + + self.assertEqual(len(index.diff(NULL_TREE, paths=filename)), 1) + self.assertEqual(len(index.diff(NULL_TREE_SHA, paths=filename)), 1) + self.assertEqual(len(index.diff(NULL_TREE, paths="missing")), 0) + + patch = index.diff(NULL_TREE, create_patch=True) + self.assertEqual(len(patch), 1) + self.assertIn(b"+# Initial file", patch[0].diff) + + with self.assertRaises(GitCommandError) as exc_info: + index.diff(NULL_TREE, bogus_option=True) + self.assertIn("usage: git diff", exc_info.exception.stderr) + def _count_existing(self, repo, files): """Return count of files that actually exist in the repository directory.""" existing = 0 @@ -546,365 +639,369 @@ def _count_existing(self, repo, files): # END num existing helper - @pytest.mark.xfail( - os.name == "nt" and Git().config("core.symlinks") == "true", - reason="Assumes symlinks are not created on Windows and opens a symlink to a nonexistent target.", - raises=FileNotFoundError, - ) @with_rw_repo("0.1.6") def test_index_mutation(self, rw_repo): - index = rw_repo.index - num_entries = len(index.entries) - cur_head = rw_repo.head - - uname = "Thomas MÃŧller" - umail = "sd@company.com" - with rw_repo.config_writer() as writer: - writer.set_value("user", "name", uname) - writer.set_value("user", "email", umail) - self.assertEqual(writer.get_value("user", "name"), uname) - - # Remove all of the files, provide a wild mix of paths, BaseIndexEntries, - # IndexEntries. - def mixed_iterator(): - count = 0 - for entry in index.entries.values(): - type_id = count % 5 - if type_id == 0: # path (str) - yield entry.path - elif type_id == 1: # path (PathLike) - yield Path(entry.path) - elif type_id == 2: # blob - yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) - elif type_id == 3: # BaseIndexEntry - yield BaseIndexEntry(entry[:4]) - elif type_id == 4: # IndexEntry - yield entry - else: - raise AssertionError("Invalid Type") - count += 1 - # END for each entry - - # END mixed iterator - deleted_files = index.remove(mixed_iterator(), working_tree=False) - assert deleted_files - self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) - self.assertEqual(len(index.entries), 0) - - # Reset the index to undo our changes. - index.reset() - self.assertEqual(len(index.entries), num_entries) - - # Remove with working copy. - deleted_files = index.remove(mixed_iterator(), working_tree=True) - assert deleted_files - self.assertEqual(self._count_existing(rw_repo, deleted_files), 0) - - # Reset everything. - index.reset(working_tree=True) - self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) - - # Invalid type. - self.assertRaises(TypeError, index.remove, [1]) - - # Absolute path. - deleted_files = index.remove([osp.join(rw_repo.working_tree_dir, "lib")], r=True) - assert len(deleted_files) > 1 - self.assertRaises(ValueError, index.remove, ["/doesnt/exists"]) - - # TEST COMMITTING - # Commit changed index. - cur_commit = cur_head.commit - commit_message = "commit default head by Frèderic Çauflâ‚Ŧ" - - new_commit = index.commit(commit_message, head=False) - assert cur_commit != new_commit - self.assertEqual(new_commit.author.name, uname) - self.assertEqual(new_commit.author.email, umail) - self.assertEqual(new_commit.committer.name, uname) - self.assertEqual(new_commit.committer.email, umail) - self.assertEqual(new_commit.message, commit_message) - self.assertEqual(new_commit.parents[0], cur_commit) - self.assertEqual(len(new_commit.parents), 1) - self.assertEqual(cur_head.commit, cur_commit) - - # Commit with other actor. - cur_commit = cur_head.commit - - my_author = Actor("Frèderic Çauflâ‚Ŧ", "author@example.com") - my_committer = Actor("Committing Frèderic Çauflâ‚Ŧ", "committer@example.com") - commit_actor = index.commit(commit_message, author=my_author, committer=my_committer) - assert cur_commit != commit_actor - self.assertEqual(commit_actor.author.name, "Frèderic Çauflâ‚Ŧ") - self.assertEqual(commit_actor.author.email, "author@example.com") - self.assertEqual(commit_actor.committer.name, "Committing Frèderic Çauflâ‚Ŧ") - self.assertEqual(commit_actor.committer.email, "committer@example.com") - self.assertEqual(commit_actor.message, commit_message) - self.assertEqual(commit_actor.parents[0], cur_commit) - self.assertEqual(len(new_commit.parents), 1) - self.assertEqual(cur_head.commit, commit_actor) - self.assertEqual(cur_head.log()[-1].actor, my_committer) - - # Commit with author_date and commit_date. - cur_commit = cur_head.commit - commit_message = "commit with dates by Avinash Sajjanshetty" - - new_commit = index.commit( - commit_message, - author_date="2006-04-07T22:13:13", - commit_date="2005-04-07T22:13:13", - ) - assert cur_commit != new_commit - print(new_commit.authored_date, new_commit.committed_date) - self.assertEqual(new_commit.message, commit_message) - self.assertEqual(new_commit.authored_date, 1144447993) - self.assertEqual(new_commit.committed_date, 1112911993) - - # Same index, no parents. - commit_message = "index without parents" - commit_no_parents = index.commit(commit_message, parent_commits=[], head=True) - self.assertEqual(commit_no_parents.message, commit_message) - self.assertEqual(len(commit_no_parents.parents), 0) - self.assertEqual(cur_head.commit, commit_no_parents) - - # same index, multiple parents. - commit_message = "Index with multiple parents\n commit with another line" - commit_multi_parent = index.commit(commit_message, parent_commits=(commit_no_parents, new_commit)) - self.assertEqual(commit_multi_parent.message, commit_message) - self.assertEqual(len(commit_multi_parent.parents), 2) - self.assertEqual(commit_multi_parent.parents[0], commit_no_parents) - self.assertEqual(commit_multi_parent.parents[1], new_commit) - self.assertEqual(cur_head.commit, commit_multi_parent) - - # Re-add all files in lib. - # Get the lib folder back on disk, but get an index without it. - index.reset(new_commit.parents[0], working_tree=True).reset(new_commit, working_tree=False) - lib_file_path = osp.join("lib", "git", "__init__.py") - assert (lib_file_path, 0) not in index.entries - assert osp.isfile(osp.join(rw_repo.working_tree_dir, lib_file_path)) - - # Directory. - entries = index.add(["lib"], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - assert len(entries) > 1 - - # Glob. - entries = index.reset(new_commit).add([osp.join("lib", "git", "*.py")], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - self.assertEqual(len(entries), 14) - - # Same file. - entries = index.reset(new_commit).add( - [osp.join(rw_repo.working_tree_dir, "lib", "git", "head.py")] * 2, - fprogress=self._fprogress_add, - ) - self._assert_entries(entries) - self.assertEqual(entries[0].mode & 0o644, 0o644) - # Would fail, test is too primitive to handle this case. - # self._assert_fprogress(entries) - self._reset_progress() - self.assertEqual(len(entries), 2) - - # Missing path. - self.assertRaises(OSError, index.reset(new_commit).add, ["doesnt/exist/must/raise"]) - - # Blob from older revision overrides current index revision. - old_blob = new_commit.parents[0].tree.blobs[0] - entries = index.reset(new_commit).add([old_blob], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - self.assertEqual(index.entries[(old_blob.path, 0)].hexsha, old_blob.hexsha) - self.assertEqual(len(entries), 1) - - # Mode 0 not allowed. - null_hex_sha = Diff.NULL_HEX_SHA - null_bin_sha = b"\0" * 20 - self.assertRaises( - ValueError, - index.reset(new_commit).add, - [BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))], - ) - - # Add new file. - new_file_relapath = "my_new_file" - self._make_file(new_file_relapath, "hello world", rw_repo) - entries = index.reset(new_commit).add( - [BaseIndexEntry((0o10644, null_bin_sha, 0, new_file_relapath))], - fprogress=self._fprogress_add, - ) - self._assert_entries(entries) - self._assert_fprogress(entries) - self.assertEqual(len(entries), 1) - self.assertNotEqual(entries[0].hexsha, null_hex_sha) - - # Add symlink. - if os.name != "nt": - for target in ("/etc/nonexisting", "/etc/passwd", "/etc"): - basename = "my_real_symlink" - - link_file = osp.join(rw_repo.working_tree_dir, basename) - os.symlink(target, link_file) - entries = index.reset(new_commit).add([link_file], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - self.assertEqual(len(entries), 1) - self.assertTrue(S_ISLNK(entries[0].mode)) - self.assertTrue(S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode)) - - # We expect only the target to be written. - self.assertEqual( - index.repo.odb.stream(entries[0].binsha).read().decode("ascii"), - target, - ) - - os.remove(link_file) - # END for each target - # END real symlink test - - # Add fake symlink and assure it checks out as a symlink. - fake_symlink_relapath = "my_fake_symlink" - link_target = "/etc/that" - fake_symlink_path = self._make_file(fake_symlink_relapath, link_target, rw_repo) - fake_entry = BaseIndexEntry((0o120000, null_bin_sha, 0, fake_symlink_relapath)) - entries = index.reset(new_commit).add([fake_entry], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - assert entries[0].hexsha != null_hex_sha - self.assertEqual(len(entries), 1) - self.assertTrue(S_ISLNK(entries[0].mode)) - - # Check that this also works with an alternate method. - full_index_entry = IndexEntry.from_base(BaseIndexEntry((0o120000, entries[0].binsha, 0, entries[0].path))) - entry_key = index.entry_key(full_index_entry) - index.reset(new_commit) - - assert entry_key not in index.entries - index.entries[entry_key] = full_index_entry - index.write() - index.update() # Force reread of entries. - new_entry = index.entries[entry_key] - assert S_ISLNK(new_entry.mode) + with xfail_if_raises( + sys.platform == "win32" and (Git().config("core.symlinks") == "true" or symlinks_supported()), + raises=(FileNotFoundError, GitCommandError), + reason="Assumes symlinks are not created on Windows and opens a symlink to a nonexistent target.", + ): + index = rw_repo.index + num_entries = len(index.entries) + cur_head = rw_repo.head + + uname = "Thomas MÃŧller" + umail = "sd@company.com" + with rw_repo.config_writer() as writer: + writer.set_value("user", "name", uname) + writer.set_value("user", "email", umail) + self.assertEqual(writer.get_value("user", "name"), uname) + + # Remove all of the files, provide a wild mix of paths, BaseIndexEntries, + # IndexEntries. + def mixed_iterator(): + count = 0 + for entry in index.entries.values(): + type_id = count % 5 + if type_id == 0: # path (str) + yield entry.path + elif type_id == 1: # path (PathLike) + yield Path(entry.path) + elif type_id == 2: # path mock (PathLike) + yield PathLikeMock(entry.path) + elif type_id == 3: # path mock in a blob + yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) + elif type_id == 4: # blob + yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) + elif type_id == 5: # BaseIndexEntry + yield BaseIndexEntry(entry[:4]) + elif type_id == 6: # IndexEntry + yield entry + else: + raise AssertionError("Invalid Type") + count += 1 + # END for each entry + + # END mixed iterator + deleted_files = index.remove(mixed_iterator(), working_tree=False) + assert deleted_files + self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) + self.assertEqual(len(index.entries), 0) + + # Reset the index to undo our changes. + index.reset() + self.assertEqual(len(index.entries), num_entries) + + # Remove with working copy. + deleted_files = index.remove(mixed_iterator(), working_tree=True) + assert deleted_files + self.assertEqual(self._count_existing(rw_repo, deleted_files), 0) + + # Reset everything. + index.reset(working_tree=True) + self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) + + # Invalid type. + self.assertRaises(TypeError, index.remove, [1]) + + # Absolute path. + deleted_files = index.remove([osp.join(rw_repo.working_tree_dir, "lib")], r=True) + assert len(deleted_files) > 1 + self.assertRaises(ValueError, index.remove, ["/doesnt/exists"]) + + # TEST COMMITTING + # Commit changed index. + cur_commit = cur_head.commit + commit_message = "commit default head by Frèderic Çauflâ‚Ŧ" + + new_commit = index.commit(commit_message, head=False) + assert cur_commit != new_commit + self.assertEqual(new_commit.author.name, uname) + self.assertEqual(new_commit.author.email, umail) + self.assertEqual(new_commit.committer.name, uname) + self.assertEqual(new_commit.committer.email, umail) + self.assertEqual(new_commit.message, commit_message) + self.assertEqual(new_commit.parents[0], cur_commit) + self.assertEqual(len(new_commit.parents), 1) + self.assertEqual(cur_head.commit, cur_commit) + + # Commit with other actor. + cur_commit = cur_head.commit + + my_author = Actor("Frèderic Çauflâ‚Ŧ", "author@example.com") + my_committer = Actor("Committing Frèderic Çauflâ‚Ŧ", "committer@example.com") + commit_actor = index.commit(commit_message, author=my_author, committer=my_committer) + assert cur_commit != commit_actor + self.assertEqual(commit_actor.author.name, "Frèderic Çauflâ‚Ŧ") + self.assertEqual(commit_actor.author.email, "author@example.com") + self.assertEqual(commit_actor.committer.name, "Committing Frèderic Çauflâ‚Ŧ") + self.assertEqual(commit_actor.committer.email, "committer@example.com") + self.assertEqual(commit_actor.message, commit_message) + self.assertEqual(commit_actor.parents[0], cur_commit) + self.assertEqual(len(new_commit.parents), 1) + self.assertEqual(cur_head.commit, commit_actor) + self.assertEqual(cur_head.log()[-1].actor, my_committer) + + # Commit with author_date and commit_date. + cur_commit = cur_head.commit + commit_message = "commit with dates by Avinash Sajjanshetty" + + new_commit = index.commit( + commit_message, + author_date="2006-04-07T22:13:13", + commit_date="2005-04-07T22:13:13", + ) + assert cur_commit != new_commit + print(new_commit.authored_date, new_commit.committed_date) + self.assertEqual(new_commit.message, commit_message) + self.assertEqual(new_commit.authored_date, 1144447993) + self.assertEqual(new_commit.committed_date, 1112911993) + + # Same index, no parents. + commit_message = "index without parents" + commit_no_parents = index.commit(commit_message, parent_commits=[], head=True) + self.assertEqual(commit_no_parents.message, commit_message) + self.assertEqual(len(commit_no_parents.parents), 0) + self.assertEqual(cur_head.commit, commit_no_parents) + + # same index, multiple parents. + commit_message = "Index with multiple parents\n commit with another line" + commit_multi_parent = index.commit(commit_message, parent_commits=(commit_no_parents, new_commit)) + self.assertEqual(commit_multi_parent.message, commit_message) + self.assertEqual(len(commit_multi_parent.parents), 2) + self.assertEqual(commit_multi_parent.parents[0], commit_no_parents) + self.assertEqual(commit_multi_parent.parents[1], new_commit) + self.assertEqual(cur_head.commit, commit_multi_parent) + + # Re-add all files in lib. + # Get the lib folder back on disk, but get an index without it. + index.reset(new_commit.parents[0], working_tree=True).reset(new_commit, working_tree=False) + lib_file_path = osp.join("lib", "git", "__init__.py") + assert (lib_file_path, 0) not in index.entries + assert osp.isfile(osp.join(rw_repo.working_tree_dir, lib_file_path)) + + # Directory. + entries = index.add(["lib"], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + assert len(entries) > 1 + + # Glob. + entries = index.reset(new_commit).add([osp.join("lib", "git", "*.py")], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + self.assertEqual(len(entries), 14) + + # Same file. + entries = index.reset(new_commit).add( + [osp.join(rw_repo.working_tree_dir, "lib", "git", "head.py")] * 2, + fprogress=self._fprogress_add, + ) + self._assert_entries(entries) + self.assertEqual(entries[0].mode & 0o644, 0o644) + # Would fail, test is too primitive to handle this case. + # self._assert_fprogress(entries) + self._reset_progress() + self.assertEqual(len(entries), 2) + + # Missing path. + self.assertRaises(OSError, index.reset(new_commit).add, ["doesnt/exist/must/raise"]) + + # Blob from older revision overrides current index revision. + old_blob = new_commit.parents[0].tree.blobs[0] + entries = index.reset(new_commit).add([old_blob], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + self.assertEqual(index.entries[(old_blob.path, 0)].hexsha, old_blob.hexsha) + self.assertEqual(len(entries), 1) + + # Mode 0 not allowed. + null_hex_sha = Diff.NULL_HEX_SHA + null_bin_sha = b"\0" * 20 + self.assertRaises( + ValueError, + index.reset(new_commit).add, + [BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))], + ) - # A tree created from this should contain the symlink. - tree = index.write_tree() - assert fake_symlink_relapath in tree - index.write() # Flush our changes for the checkout. - - # Check out the fake link, should be a link then. - assert not S_ISLNK(os.stat(fake_symlink_path)[ST_MODE]) - os.remove(fake_symlink_path) - index.checkout(fake_symlink_path) - - # On Windows, we currently assume we will never get symlinks. - if os.name == "nt": - # Symlinks should contain the link as text (which is what a - # symlink actually is). - with open(fake_symlink_path, "rt") as fd: - self.assertEqual(fd.read(), link_target) - else: - self.assertTrue(S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE])) - - # TEST RENAMING - def assert_mv_rval(rval): - for source, dest in rval: - assert not osp.exists(source) and osp.exists(dest) - # END for each renamed item - - # END move assertion utility - - self.assertRaises(ValueError, index.move, ["just_one_path"]) - # Try to move a file onto an existing file. - files = ["AUTHORS", "LICENSE"] - self.assertRaises(GitCommandError, index.move, files) - - # Again, with force. - assert_mv_rval(index.move(files, f=True)) - - # Move files into a directory - dry run. - paths = ["LICENSE", "VERSION", "doc"] - rval = index.move(paths, dry_run=True) - self.assertEqual(len(rval), 2) - assert osp.exists(paths[0]) - - # Again, no dry run. - rval = index.move(paths) - assert_mv_rval(rval) - - # Move dir into dir. - rval = index.move(["doc", "test"]) - assert_mv_rval(rval) - - # TEST PATH REWRITING - ###################### - count = [0] - - def rewriter(entry): - rval = str(count[0]) - count[0] += 1 - return rval - - # END rewriter - - def make_paths(): - """Help out the test by yielding two existing paths and one new path.""" - yield "CHANGES" - yield "ez_setup.py" - yield index.entries[index.entry_key("README", 0)] - yield index.entries[index.entry_key(".gitignore", 0)] - - for fid in range(3): - fname = "newfile%i" % fid - with open(fname, "wb") as fd: - fd.write(b"abcd") - yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0o100644, fname) - # END for each new file - - # END path producer - paths = list(make_paths()) - self._assert_entries(index.add(paths, path_rewriter=rewriter)) - - for filenum in range(len(paths)): - assert index.entry_key(str(filenum), 0) in index.entries - - # TEST RESET ON PATHS - ###################### - arela = "aa" - brela = "bb" - afile = self._make_file(arela, "adata", rw_repo) - bfile = self._make_file(brela, "bdata", rw_repo) - akey = index.entry_key(arela, 0) - bkey = index.entry_key(brela, 0) - keys = (akey, bkey) - absfiles = (afile, bfile) - files = (arela, brela) - - for fkey in keys: - assert fkey not in index.entries - - index.add(files, write=True) - nc = index.commit("2 files committed", head=False) - - for fkey in keys: - assert fkey in index.entries - - # Just the index. - index.reset(paths=(arela, afile)) - assert akey not in index.entries - assert bkey in index.entries - - # Now with working tree - files on disk as well as entries must be recreated. - rw_repo.head.commit = nc - for absfile in absfiles: - os.remove(absfile) - - index.reset(working_tree=True, paths=files) - - for fkey in keys: - assert fkey in index.entries - for absfile in absfiles: - assert osp.isfile(absfile) + # Add new file. + new_file_relapath = "my_new_file" + self._make_file(new_file_relapath, "hello world", rw_repo) + entries = index.reset(new_commit).add( + [BaseIndexEntry((0o10644, null_bin_sha, 0, new_file_relapath))], + fprogress=self._fprogress_add, + ) + self._assert_entries(entries) + self._assert_fprogress(entries) + self.assertEqual(len(entries), 1) + self.assertNotEqual(entries[0].hexsha, null_hex_sha) + + # Add symlink. + if sys.platform != "win32": + for target in ("/etc/nonexisting", "/etc/passwd", "/etc"): + basename = "my_real_symlink" + + link_file = osp.join(rw_repo.working_tree_dir, basename) + os.symlink(target, link_file) + entries = index.reset(new_commit).add([link_file], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + self.assertEqual(len(entries), 1) + self.assertTrue(S_ISLNK(entries[0].mode)) + self.assertTrue(S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode)) + + # We expect only the target to be written. + self.assertEqual( + index.repo.odb.stream(entries[0].binsha).read().decode("ascii"), + target, + ) + + os.remove(link_file) + # END for each target + # END real symlink test + + # Add fake symlink and assure it checks out as a symlink. + fake_symlink_relapath = "my_fake_symlink" + link_target = "/etc/that" + fake_symlink_path = self._make_file(fake_symlink_relapath, link_target, rw_repo) + fake_entry = BaseIndexEntry((0o120000, null_bin_sha, 0, fake_symlink_relapath)) + entries = index.reset(new_commit).add([fake_entry], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + assert entries[0].hexsha != null_hex_sha + self.assertEqual(len(entries), 1) + self.assertTrue(S_ISLNK(entries[0].mode)) + + # Check that this also works with an alternate method. + full_index_entry = IndexEntry.from_base(BaseIndexEntry((0o120000, entries[0].binsha, 0, entries[0].path))) + entry_key = index.entry_key(full_index_entry) + index.reset(new_commit) + + assert entry_key not in index.entries + index.entries[entry_key] = full_index_entry + index.write() + index.update() # Force reread of entries. + new_entry = index.entries[entry_key] + assert S_ISLNK(new_entry.mode) + + # A tree created from this should contain the symlink. + tree = index.write_tree() + assert fake_symlink_relapath in tree + index.write() # Flush our changes for the checkout. + + # Check out the fake link, should be a link then. + assert not S_ISLNK(os.stat(fake_symlink_path)[ST_MODE]) + os.remove(fake_symlink_path) + index.checkout(fake_symlink_path) + + # On Windows, we currently assume we will never get symlinks. + if sys.platform == "win32": + # Symlinks should contain the link as text (which is what a + # symlink actually is). + with open(fake_symlink_path, "rt") as fd: + self.assertEqual(fd.read(), link_target) + else: + self.assertTrue(S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE])) + + # TEST RENAMING + def assert_mv_rval(rval): + for source, dest in rval: + assert not osp.exists(source) and osp.exists(dest) + # END for each renamed item + + # END move assertion utility + + self.assertRaises(ValueError, index.move, ["just_one_path"]) + # Try to move a file onto an existing file. + files = ["AUTHORS", "LICENSE"] + self.assertRaises(GitCommandError, index.move, files) + + # Again, with force. + assert_mv_rval(index.move(files, f=True)) + + # Move files into a directory - dry run. + paths = ["LICENSE", "VERSION", "doc"] + rval = index.move(paths, dry_run=True) + self.assertEqual(len(rval), 2) + assert osp.exists(paths[0]) + + # Again, no dry run. + rval = index.move(paths) + assert_mv_rval(rval) + + # Move dir into dir. + rval = index.move(["doc", "test"]) + assert_mv_rval(rval) + + # TEST PATH REWRITING + ###################### + count = [0] + + def rewriter(entry): + rval = str(count[0]) + count[0] += 1 + return rval + + # END rewriter + + def make_paths(): + """Help out the test by yielding two existing paths and one new path.""" + yield "CHANGES" + yield "ez_setup.py" + yield index.entries[index.entry_key("README", 0)] + yield index.entries[index.entry_key(".gitignore", 0)] + + for fid in range(3): + fname = "newfile%i" % fid + with open(fname, "wb") as fd: + fd.write(b"abcd") + yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0o100644, fname) + # END for each new file + + # END path producer + paths = list(make_paths()) + self._assert_entries(index.add(paths, path_rewriter=rewriter)) + + for filenum in range(len(paths)): + assert index.entry_key(str(filenum), 0) in index.entries + + # TEST RESET ON PATHS + ###################### + arela = "aa" + brela = "bb" + afile = self._make_file(arela, "adata", rw_repo) + bfile = self._make_file(brela, "bdata", rw_repo) + akey = index.entry_key(arela, 0) + bkey = index.entry_key(brela, 0) + keys = (akey, bkey) + absfiles = (afile, bfile) + files = (arela, brela) + + for fkey in keys: + assert fkey not in index.entries + + index.add(files, write=True) + nc = index.commit("2 files committed", head=False) + + for fkey in keys: + assert fkey in index.entries + + # Just the index. + index.reset(paths=(arela, afile)) + assert akey not in index.entries + assert bkey in index.entries + + # Now with working tree - files on disk as well as entries must be recreated. + rw_repo.head.commit = nc + for absfile in absfiles: + os.remove(absfile) + + index.reset(working_tree=True, paths=files) + + for fkey in keys: + assert fkey in index.entries + for absfile in absfiles: + assert osp.isfile(absfile) @with_rw_repo("HEAD") def test_compare_write_tree(self, rw_repo): @@ -942,10 +1039,10 @@ def test_index_new(self): @with_rw_repo("HEAD", bare=True) def test_index_bare_add(self, rw_bare_repo): - # Something is wrong after cloning to a bare repo, reading the - # property rw_bare_repo.working_tree_dir will return '/tmp' - # instead of throwing the Exception we are expecting. This is - # a quick hack to make this test fail when expected. + # Something is wrong after cloning to a bare repo, reading the property + # rw_bare_repo.working_tree_dir will return '/tmp' instead of throwing the + # Exception we are expecting. This is a quick hack to make this test fail when + # expected. assert rw_bare_repo.working_tree_dir is None assert rw_bare_repo.bare contents = b"This is a BytesIO file" @@ -970,7 +1067,8 @@ def test_index_bare_add(self, rw_bare_repo): @with_rw_directory def test_add_utf8P_path(self, rw_dir): - # NOTE: fp is not a Unicode object in Python 2 (which is the source of the problem). + # NOTE: fp is not a Unicode object in Python 2 + # (which is the source of the problem). fp = osp.join(rw_dir, "ø.txt") with open(fp, "wb") as fs: fs.write("content of ø".encode("utf-8")) @@ -1005,10 +1103,69 @@ class Mocked: rel = index._to_relative_path(path) self.assertEqual(rel, os.path.relpath(path, root)) + def test__to_relative_path_absolute_trailing_slash(self): + repo_root = os.path.join(osp.abspath(os.sep), "directory1", "repo_root") + + class Mocked: + bare = False + git_dir = repo_root + working_tree_dir = repo_root + + repo = Mocked() + path = os.path.join(repo_root, f"directory2{os.sep}") + index = IndexFile(repo) + + expected_path = "directory2/" + actual_path = index._to_relative_path(path) + self.assertEqual(expected_path, actual_path) + + @pytest.mark.skipif(sys.platform != "win32", reason="Specifically for Windows.") + def test__to_relative_path_windows_unc_share_root(self): + for repo_root in [R"\\server\share", R"\\?\UNC\server\share"]: + with self.subTest(repo_root=repo_root): + repo = mock.Mock(bare=False, git_dir=repo_root, working_tree_dir=repo_root) + index = IndexFile(repo) + + self.assertEqual(index._to_relative_path(repo_root), ".") + + @pytest.mark.skipif(sys.platform != "win32", reason="Specifically for Windows.") + @with_rw_directory + def test__to_relative_path_windows_path_kinds(self, rw_dir): + repo_root = osp.join(rw_dir, "repo") + os.makedirs(osp.join(repo_root, "nested")) + + class Mocked: + bare = False + git_dir = osp.join(repo_root, ".git") + working_tree_dir = repo_root + + index = IndexFile(Mocked()) + inside_path = osp.join(repo_root, "nested", "file") + _drive, rooted_inside_path = osp.splitdrive(inside_path) + + self.assertEqual(index._to_relative_path(inside_path), "nested/file") + self.assertEqual(index._to_relative_path(rooted_inside_path), "nested/file") + self.assertEqual(index._to_relative_path(rooted_inside_path.replace("\\", "/")), "nested/file") + self.assertEqual(index._to_relative_path(PathLikeMock(inside_path)), "nested/file") + self.assertEqual(index._to_relative_path(inside_path.upper()), "NESTED/FILE") + self.assertRaises(ValueError, index._to_relative_path, osp.join(repo_root + "-other", "file")) + self.assertRaises(ValueError, index._to_relative_path, osp.join("..", "outside")) + self.assertRaises(ValueError, index._to_relative_path, f"{osp.splitdrive(repo_root)[0]}relative") + self.assertRaises(ValueError, index._to_relative_path, R"Z:\outside") + self.assertRaises(ValueError, index._to_relative_path, R"\\server\share\outside") + self.assertRaises(ValueError, index._to_relative_path, R"\\?\C:\outside") + + Mocked.bare = True + bare_index = IndexFile(Mocked()) + self.assertRaises(InvalidGitRepositoryError, bare_index._to_relative_path, rooted_inside_path) + self.assertRaises( + InvalidGitRepositoryError, bare_index._to_relative_path, f"{osp.splitdrive(repo_root)[0]}relative" + ) + @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, @@ -1019,16 +1176,106 @@ class Mocked: def test_run_commit_hook(self, rw_repo): index = rw_repo.index _make_hook(index.repo.git_dir, "fake-hook", "echo 'ran fake hook' >output.txt") - run_commit_hook("fake-hook", index) - output = Path(rw_repo.git_dir, "output.txt").read_text(encoding="utf-8") - self.assertEqual(output, "ran fake hook\n") + output = Path(rw_repo.git_dir, "output.txt") + with mock.patch.object(Repo, "config_level", ("repository",)): + with mock.patch.object(Git, "execute", side_effect=AssertionError("hook lookup must not run git")): + run_commit_hook("fake-hook", index) + self.assertEqual(output.read_text(encoding="utf-8"), "ran fake hook\n") + + output.unlink() + with index.repo.config_writer() as writer: + writer.set_value("core", "hooksPath", "") + run_commit_hook("fake-hook", index) + + self.assertEqual(output.read_text(encoding="utf-8"), "ran fake hook\n") + + @with_rw_directory + def test_run_commit_hook_outside_worktree_on_windows(self, rw_dir): + root = Path(rw_dir).resolve() + repo = Repo.init(root / "repo") + hooks_dir = root / "hooks" + _make_hook(root, "fake-hook", "exit 0") + system_root = root / "Windows" + system_bash = system_root / "System32" / "bash.exe" + git_executable = root / "Git" / "cmd" / "git.exe" + git_bash = root / "Git" / "bin" / "bash.exe" + for executable in (system_bash, git_executable, git_bash): + executable.parent.mkdir(parents=True) + executable.touch() + executable.chmod(0o755) + with repo.config_writer() as writer: + writer.set_value("core", "hooksPath", str(hooks_dir)) + + # Model a normal Windows PATH: System32 (containing the WSL launcher) comes + # before Git's cmd directory, while Git's Bash is not itself on PATH. This + # exercises both Git-installation discovery and shell selection without + # mocking either resolver's answer. + with mock.patch("git.index.fun.sys.platform", "win32"), mock.patch.object( + Git, "GIT_PYTHON_GIT_EXECUTABLE", "git" + ), mock.patch.dict(os.environ, {"SystemRoot": str(system_root)}), mock.patch( + "git.index.fun.os.get_exec_path", return_value=["", str(system_bash.parent), str(git_executable.parent)] + ): + with mock.patch("git.index.fun.safer_popen") as popen, mock.patch("git.index.fun.handle_process_output"): + popen.return_value.returncode = 0 + run_commit_hook("fake-hook", repo.index) + + command = popen.call_args[0][0] + self.assertEqual(command, [str(git_bash), "../hooks/fake-hook"]) + + @with_rw_directory + def test_windows_bash_lookup_respects_explicit_current_directory_in_path(self, rw_dir): + root = Path(rw_dir).resolve() + bash = root / "bash.exe" + bash.touch() + bash.chmod(0o755) + + # An explicitly listed directory is trusted PATH configuration, even when + # it happens to be the current directory. This differs from an empty entry, + # which Windows requires PATH lookup to ignore. + with cwd(root), mock.patch("git.index.fun.os.get_exec_path", return_value=[str(root)]): + self.assertEqual(_which_from_path("bash.exe"), str(bash)) + + @with_rw_directory + def test_windows_bash_lookup_from_explicit_git_bin(self, rw_dir): + git_root = Path(rw_dir).resolve() / "Git" + git_executable = git_root / "bin" / "git.exe" + bash = git_root / "bin" / "bash.exe" + git_executable.parent.mkdir(parents=True) + for executable in (git_executable, bash): + executable.touch() + executable.chmod(0o755) + + # A relative executable containing a directory is resolved by CreateProcess + # from the parent process cwd, not the separately supplied child cwd. Enter the + # temporary root first because Windows cannot express a relative path between + # drives, and CI may keep the checkout and its temporary directory on different + # drives. + with cwd(Path(rw_dir).resolve()): + relative_git = osp.relpath(git_executable, os.curdir) + with mock.patch.object(Git, "GIT_PYTHON_GIT_EXECUTABLE", relative_git): + self.assertEqual(_git_for_windows_bash(), str(bash)) + + @with_rw_directory + def test_windows_bash_lookup_ignores_custom_git_executable(self, rw_dir): + root = Path(rw_dir).resolve() + for directory_name in ("cmd", "bin"): + executable = root / directory_name / "mygit.exe" + bash = root / "bin" / "bash.exe" + executable.parent.mkdir(parents=True, exist_ok=True) + bash.parent.mkdir(parents=True, exist_ok=True) + executable.touch() + bash.touch() + executable.chmod(0o755) + bash.chmod(0o755) + with mock.patch.object(Git, "GIT_PYTHON_GIT_EXECUTABLE", str(executable)): + self.assertIsNone(_git_for_windows_bash()) @ddt.data((False,), (True,)) @with_rw_directory def test_hook_uses_shell_not_from_cwd(self, rw_dir, case): (chdir_to_repo,) = case - shell_name = "bash.exe" if os.name == "nt" else "sh" + shell_name = "bash.exe" if sys.platform == "win32" else "sh" maybe_chdir = cwd(rw_dir) if chdir_to_repo else contextlib.nullcontext() repo = Repo.init(rw_dir) @@ -1067,7 +1314,7 @@ def test_hook_uses_shell_not_from_cwd(self, rw_dir, case): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, @@ -1080,6 +1327,32 @@ def test_pre_commit_hook_success(self, rw_repo): _make_hook(index.repo.git_dir, "pre-commit", "exit 0") index.commit("This should not fail") + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + raises=HookExecutionError, + ) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.WslNoDistro, + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", + raises=HookExecutionError, + ) + @with_rw_repo("HEAD") + def test_pre_commit_hook_respects_core_hooks_path(self, rw_repo): + index = rw_repo.index + hooks_dir = Path(index.repo.working_dir, "custom-hooks") + hooks_dir.mkdir() + hp = hooks_dir / "pre-commit" + hp.write_text(HOOKS_SHEBANG + "echo 'ran custom hook' >custom-hook-output.txt", encoding="utf-8") + os.chmod(hp, 0o744) + + with index.repo.config_writer() as writer: + writer.set_value("core", "hooksPath", "custom-hooks") + + index.commit("This should run the custom hook") + output = Path(rw_repo.working_dir, "custom-hook-output.txt").read_text(encoding="utf-8") + self.assertEqual(output, "ran custom hook\n") + @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", @@ -1110,7 +1383,7 @@ def test_pre_commit_hook_fail(self, rw_repo): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Wsl, @@ -1163,7 +1436,7 @@ def test_commit_msg_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @with_rw_repo("HEAD") - def test_index_add_pathlike(self, rw_repo): + def test_index_add_pathlib(self, rw_repo): git_dir = Path(rw_repo.git_dir) file = git_dir / "file.txt" @@ -1171,6 +1444,69 @@ def test_index_add_pathlike(self, rw_repo): rw_repo.index.add(file) + @with_rw_repo("HEAD") + def test_index_add_pathlike(self, rw_repo): + git_dir = Path(rw_repo.git_dir) + + file = git_dir / "file.txt" + file.touch() + + rw_repo.index.add(PathLikeMock(str(file))) + + @with_rw_repo("HEAD") + def test_index_add_non_normalized_path(self, rw_repo): + git_dir = Path(rw_repo.git_dir) + + file = git_dir / "file.txt" + file.touch() + non_normalized_path = file.as_posix() + if os.name != "nt": + non_normalized_path = "/" + non_normalized_path[1:].replace("/", "//") + + rw_repo.index.add(non_normalized_path) + + def test_index_file_v3(self): + index = IndexFile(self.rorepo, fixture_path("index_extended_flags")) + assert index.entries + assert index.version == 3 + assert len(index.entries) == 4 + assert index.entries[("init.t", 0)].skip_worktree + + # Write the data - it must match the original. + with tempfile.NamedTemporaryFile() as tmpfile: + index.write(tmpfile.name) + assert Path(tmpfile.name).read_bytes() == Path(fixture_path("index_extended_flags")).read_bytes() + + @with_rw_directory + def test_index_file_v3_with_git_command(self, tmp_dir): + tmp_dir = Path(tmp_dir) + with cwd(tmp_dir): + git = Git(tmp_dir) + git.init() + + file = tmp_dir / "file.txt" + file.write_text("hello") + git.add("--intent-to-add", "file.txt") # intent-to-add sets extended flag + + repo = Repo(tmp_dir) + index = repo.index + + assert len(index.entries) == 1 + assert index.version == 3 + entry = list(index.entries.values())[0] + assert entry.path == "file.txt" + assert entry.intent_to_add + + file2 = tmp_dir / "file2.txt" + file2.write_text("world") + index.add(["file2.txt"]) + index.write() + + status_str = git.status(porcelain=True) + status_lines = status_str.splitlines() + assert " A file.txt" in status_lines + assert "A file2.txt" in status_lines + class TestIndexUtils: @pytest.mark.parametrize("file_path_type", [str, Path]) diff --git a/test/test_installation.py b/test/test_installation.py index 15ed5b13b..e8956d5cf 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -2,59 +2,34 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import ast +import functools import os import subprocess -from test.lib import TestBase, VirtualEnvironment, with_rw_directory +from test.lib import TestBase, VirtualEnvironment, requires_symlinks, with_rw_directory class TestInstallation(TestBase): + @requires_symlinks @with_rw_directory def test_installation(self, rw_dir): - venv = self._set_up_venv(rw_dir) + venv, run = self._set_up_venv(rw_dir) - result = subprocess.run( - [venv.pip, "install", "."], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Can't install project", - ) + for project in ("./smmap", "./gitdb", "."): + result = run([venv.pip, "install", project]) + self._check_result(result, f"Can't install {project}") - result = subprocess.run( - [venv.python, "-c", "import git"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Self-test failed", - ) + result = run([venv.python, "-c", "import git"]) + self._check_result(result, "Self-test failed") - result = subprocess.run( - [venv.python, "-c", "import gitdb; import smmap"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Dependencies not installed", - ) + result = run([venv.python, "-c", "import gitdb; import smmap"]) + self._check_result(result, "Dependencies not installed") - # Even IF gitdb or any other dependency is supplied during development - # by inserting its location into PYTHONPATH or otherwise patched into - # sys.path, make sure it is not wrongly inserted as the *first* entry. - result = subprocess.run( - [venv.python, "-c", "import sys; import git; print(sys.path)"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) - syspath = result.stdout.decode("utf-8").splitlines()[0] + # Even IF gitdb or any other dependency is supplied during development by + # inserting its location into PYTHONPATH or otherwise patched into sys.path, + # make sure it is not wrongly inserted as the *first* entry. + result = run([venv.python, "-c", "import sys; import git; print(sys.path)"]) + syspath = result.stdout.splitlines()[0] syspath = ast.literal_eval(syspath) self.assertEqual( "", @@ -64,10 +39,37 @@ def test_installation(self, rw_dir): @staticmethod def _set_up_venv(rw_dir): + # Initialize the virtual environment. venv = VirtualEnvironment(rw_dir, with_pip=True) + + # Make its src directory a symlink to our own top-level source tree. os.symlink( os.path.dirname(os.path.dirname(__file__)), venv.sources, target_is_directory=True, ) - return venv + + # Create a convenience function to run commands in it. + run = functools.partial( + subprocess.run, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=venv.sources, + env={**os.environ, "PYTHONWARNINGS": "error"}, + ) + + return venv, run + + def _check_result(self, result, failure_summary): + self.assertEqual( + 0, + result.returncode, + msg=self._prepare_failure_message(result, failure_summary), + ) + + @staticmethod + def _prepare_failure_message(result, failure_summary): + stdout = result.stdout.rstrip() + stderr = result.stderr.rstrip() + return f"{failure_summary}\n\nstdout:\n{stdout}\n\nstderr:\n{stderr}" diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 4ef75f4aa..98658e02f 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -173,7 +173,7 @@ def test_cloned_repo_object(self, local_dir): # [15-test_cloned_repo_object] def print_files_from_git(root, level=0): for entry in root: - print(f'{"-" * 4 * level}| {entry.path}, {entry.type}') + print(f"{'-' * 4 * level}| {entry.path}, {entry.type}") if entry.type == "tree": print_files_from_git(entry, level + 1) diff --git a/test/test_reflog.py b/test/test_reflog.py index 1bd2e5dab..7ce64219a 100644 --- a/test/test_reflog.py +++ b/test/test_reflog.py @@ -5,9 +5,10 @@ import tempfile from git.objects import IndexObject -from git.refs import RefLogEntry, RefLog +from git.refs import RefLog, RefLogEntry +from git.util import Actor, hex_to_bin, rmtree + from test.lib import TestBase, fixture_path -from git.util import Actor, rmtree, hex_to_bin class TestRefLog(TestBase): diff --git a/test/test_refs.py b/test/test_refs.py index 2656ceab1..32c8fbe34 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -3,31 +3,48 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import contextlib from itertools import chain +import os.path as osp from pathlib import Path +import tempfile + +from gitdb.exc import BadName from git import ( - Reference, - Head, - TagReference, - RemoteReference, Commit, - SymbolicReference, GitCommandError, - RefLog, GitConfigParser, + Head, + RefLog, + Reference, + RemoteReference, + Repo, + SymbolicReference, + TagReference, ) +from git.exc import UnsafeOptionError from git.objects.tag import TagObject -from test.lib import TestBase, with_rw_repo -from git.util import Actor -from gitdb.exc import BadName - import git.refs as refs -import os.path as osp -import tempfile +from git.util import Actor, rmtree + +from test.lib import TestBase, requires_symlinks, with_rw_repo, PathLikeMock class TestRefs(TestBase): + @contextlib.contextmanager + def _repo_with_initial_commit(self, base_dir): + repo_dir = base_dir / "repo" + repo = Repo.init(repo_dir) + (repo_dir / "file.txt").write_text("initial\n", encoding="utf-8") + repo.index.add(["file.txt"]) + repo.index.commit("initial") + try: + yield repo + finally: + repo.git.clear_cache() + rmtree(repo_dir) + def test_from_path(self): # Should be able to create any reference directly. for ref_type in (Reference, Head, TagReference, RemoteReference): @@ -42,6 +59,52 @@ def test_from_path(self): self.assertRaises(ValueError, TagReference, self.rorepo, "refs/invalid/tag") # Works without path check. TagReference(self.rorepo, "refs/invalid/tag", check_path=False) + # Check remoteness + assert Reference(self.rorepo, "refs/remotes/origin").is_remote() + + @with_rw_repo("HEAD") + def test_tag_create_rejects_unsafe_file_options(self, rw_repo): + with tempfile.NamedTemporaryFile("w", encoding="utf-8") as message: + message.write("private tag message") + message.flush() + for index, option in enumerate(({"F": message.name}, {"file": message.name})): + with self.assertRaises(UnsafeOptionError): + TagReference.create(rw_repo, f"unsafe-{index}", **option) + + for args in ( + ("unsafe-reference", f"--file={message.name}"), + (f"--file={message.name}", "HEAD"), + (f"-eF{message.name}", "HEAD"), + (f"-iF{message.name}", "HEAD"), + ): + with self.assertRaises(UnsafeOptionError): + TagReference.create(rw_repo, *args) + + with self.assertRaises(UnsafeOptionError): + TagReference.create(rw_repo, "unsafe-ref-kwarg", ref=f"--file={message.name}") + + tag = TagReference.create(rw_repo, "legacy-ref", ref="HEAD", allow_unsafe_options=True) + self.assertEqual(tag.commit, rw_repo.head.commit) + + tag = TagReference.create(rw_repo, "allowed-file", F=message.name, allow_unsafe_options=True) + self.assertEqual(tag.tag.message, "private tag message") + + def test_from_pathlike(self): + # Should be able to create any reference directly. + for ref_type in (Reference, Head, TagReference, RemoteReference): + for name in ("rela_name", "path/rela_name"): + full_path = ref_type.to_full_path(PathLikeMock(name)) + instance = ref_type.from_path(self.rorepo, PathLikeMock(full_path)) + assert isinstance(instance, ref_type) + # END for each name + # END for each type + + # Invalid path. + self.assertRaises(ValueError, TagReference, self.rorepo, "refs/invalid/tag") + # Works without path check. + TagReference(self.rorepo, PathLikeMock("refs/invalid/tag"), check_path=False) + # Check remoteness + assert Reference(self.rorepo, PathLikeMock("refs/remotes/origin")).is_remote() def test_tag_base(self): tag_object_refs = [] @@ -211,10 +274,55 @@ def test_orig_head(self): @with_rw_repo("0.1.6") def test_head_checkout_detached_head(self, rw_repo): - res = rw_repo.remotes.origin.refs.master.checkout() + res = rw_repo.remotes.origin.refs.HEAD.reference.checkout() assert isinstance(res, SymbolicReference) assert res.name == "HEAD" + @with_rw_repo("HEAD") + def test_head_checkout_rejects_pathspec_from_file(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + pathspecs = Path(tdir) / "pathspecs" + pathspecs.write_bytes(b"unmatched-path-one\nunmatched-path-two") + for option_name in ("pathspec_from_file", "pathspec_from"): + with self.assertRaises(UnsafeOptionError): + rw_repo.active_branch.checkout( + pathspec_file_nul=True, + **{option_name: str(pathspecs)}, + ) + + @with_rw_repo("HEAD") + def test_head_reset_rejects_pathspec_from_file(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + pathspecs = Path(tdir) / "pathspecs" + pathspecs.write_bytes(b"unmatched-path-one\nunmatched-path-two") + for option_name in ("pathspec_from_file", "pathspec_from"): + with self.assertRaises(UnsafeOptionError): + rw_repo.head.reset( + pathspec_file_nul=True, + **{option_name: str(pathspecs)}, + ) + for option_name in ("--pathspec-from-file", "--pathspec-from"): + with self.assertRaises(UnsafeOptionError): + rw_repo.head.reset( + f"{option_name}={pathspecs}", + pathspec_file_nul=True, + ) + + @with_rw_repo("HEAD") + def test_head_commands_allow_explicit_pathspec_from_file(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + pathspecs = Path(tdir) / "pathspecs" + pathspecs.write_bytes(b"CHANGES\0") + options = { + "pathspec_from_file": str(pathspecs), + "pathspec_file_nul": True, + "allow_unsafe_options": True, + } + head = rw_repo.head + branch = rw_repo.active_branch + assert head.reset(**options) is head + assert branch.checkout(**options) == branch + @with_rw_repo("0.1.6") def test_head_reset(self, rw_repo): cur_head = rw_repo.head @@ -245,8 +353,8 @@ def test_head_reset(self, rw_repo): cur_head.reset(new_head_commit) rw_repo.index.checkout(["lib"], force=True) - # Now that we have a write write repo, change the HEAD reference - it's - # like "git-reset --soft". + # Now that we have a write write repo, change the HEAD reference - it's like + # "git-reset --soft". heads = rw_repo.heads assert heads for head in heads: @@ -349,8 +457,8 @@ def test_head_reset(self, rw_repo): for remote in remotes: refs = remote.refs - # If a HEAD exists, it must be deleted first. Otherwise it might - # end up pointing to an invalid ref it the ref was deleted before. + # If a HEAD exists, it must be deleted first. Otherwise it might end up + # pointing to an invalid ref it the ref was deleted before. remote_head_name = "HEAD" if remote_head_name in refs: RemoteReference.delete(rw_repo, refs[remote_head_name]) @@ -383,7 +491,7 @@ def test_head_reset(self, rw_repo): # Setting a non-commit as commit fails, but succeeds as object. head_tree = head.commit.tree self.assertRaises(ValueError, setattr, head, "commit", head_tree) - assert head.commit == old_commit # and the ref did not change + assert head.commit == old_commit # And the ref did not change. # We allow heads to point to any object. head.object = head_tree assert head.object == head_tree @@ -492,8 +600,8 @@ def test_head_reset(self, rw_repo): # Would raise if the symref wouldn't have been deleted (probably). symref = SymbolicReference.create(rw_repo, symref_path, cur_head.reference) - # Test symbolic references which are not at default locations like HEAD - # or FETCH_HEAD - they may also be at spots in refs of course. + # Test symbolic references which are not at default locations like HEAD or + # FETCH_HEAD - they may also be at spots in refs of course. symbol_ref_path = "refs/symbol_ref" symref = SymbolicReference(rw_repo, symbol_ref_path) assert symref.path == symbol_ref_path @@ -525,14 +633,13 @@ def test_head_reset(self, rw_repo): assert active_branch in heads assert rw_repo.tags - # We should be able to iterate all symbolic refs as well - in that case - # we should expect only symbolic references to be returned. + # We should be able to iterate all symbolic refs as well - in that case we + # should expect only symbolic references to be returned. for symref in SymbolicReference.iter_items(rw_repo): assert not symref.is_detached - # When iterating references, we can get references and symrefs - # when deleting all refs, I'd expect them to be gone! Even from - # the packed ones. + # When iterating references, we can get references and symrefs when deleting all + # refs, I'd expect them to be gone! Even from the packed ones. # For this to work, we must not be on any branch. rw_repo.head.reference = rw_repo.head.commit deleted_refs = set() @@ -577,9 +684,9 @@ def test_head_reset(self, rw_repo): self.assertRaises(ValueError, setattr, ref, "commit", "nonsense") assert not ref.is_valid() - # I am sure I had my reason to make it a class method at first, but - # now it doesn't make so much sense anymore, want an instance method as well - # See http://byronimo.lighthouseapp.com/projects/51787-gitpython/tickets/27 + # I am sure I had my reason to make it a class method at first, but now it + # doesn't make so much sense anymore, want an instance method as well. See: + # http://byronimo.lighthouseapp.com/projects/51787-gitpython/tickets/27 Reference.delete(ref.repo, ref.path) assert not ref.is_valid() @@ -615,12 +722,12 @@ def test_dereference_recursive(self): assert SymbolicReference.dereference_recursive(self.rorepo, "HEAD") def test_reflog(self): - assert isinstance(self.rorepo.heads.master.log(), RefLog) + assert isinstance(self.rorepo.active_branch.log(), RefLog) def test_refs_outside_repo(self): # Create a file containing a valid reference outside the repository. Attempting - # to access it should raise an exception, due to it containing a parent directory - # reference ('..'). This tests for CVE-2023-41040. + # to access it should raise an exception, due to it containing a parent + # directory reference ('..'). This tests for CVE-2023-41040. git_dir = Path(self.rorepo.git_dir) repo_parent_dir = git_dir.parent.parent with tempfile.NamedTemporaryFile(dir=repo_parent_dir) as ref_file: @@ -629,38 +736,160 @@ def test_refs_outside_repo(self): ref_file_name = Path(ref_file.name).name self.assertRaises(BadName, self.rorepo.commit, f"../../{ref_file_name}") + def test_reference_create_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_write.txt" + + self.assertRaises(ValueError, Reference.create, repo, "../../../outside_write.txt", "HEAD") + assert not outside_path.exists() + + def test_symbolic_reference_create_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_write.txt" + + self.assertRaises(ValueError, SymbolicReference.create, repo, "../../outside_write.txt", "HEAD") + assert not outside_path.exists() + + def test_symbolic_reference_set_reference_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_write.txt" + + self.assertRaises(ValueError, SymbolicReference(repo, "../../outside_write.txt").set_reference, "HEAD") + assert not outside_path.exists() + + def test_symbolic_reference_rename_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_move.txt" + ref = SymbolicReference.create(repo, "SAFE_RENAME_SOURCE", "HEAD") + + self.assertRaises(ValueError, ref.rename, "../../outside_move.txt") + assert not outside_path.exists() + assert Path(ref.abspath).is_file() + + def test_symbolic_reference_delete_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_delete.txt" + outside_path.write_text("do not delete\n", encoding="utf-8") + + self.assertRaises(ValueError, SymbolicReference.delete, repo, "../../outside_delete.txt") + assert outside_path.read_text(encoding="utf-8") == "do not delete\n" + + def test_symbolic_reference_log_append_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_reflog.txt" + + ref = SymbolicReference(repo, "../../../outside_reflog.txt") + self.assertRaises( + ValueError, ref.log_append, Commit.NULL_BIN_SHA, "do not write", repo.head.commit.binsha + ) + assert not outside_path.exists() + + @requires_symlinks + def test_symbolic_reference_set_reference_rejects_symlink_escape(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + with self._repo_with_initial_commit(base_dir) as repo: + outside_dir = base_dir / "outside_refs" + outside_dir.mkdir() + outside_path = outside_dir / "escaped" + + refs_heads_dir = Path(repo.common_dir) / "refs" / "heads" + refs_heads_dir.mkdir(parents=True, exist_ok=True) + symlink_path = refs_heads_dir / "link_out" + symlink_path.symlink_to(outside_dir, target_is_directory=True) + if osp.realpath(symlink_path / "escaped") == osp.abspath(symlink_path / "escaped"): + self.skipTest("realpath does not resolve directory symlinks on this platform") + + ref = SymbolicReference(repo, "refs/heads/link_out/escaped") + self.assertRaises(ValueError, ref.set_reference, "HEAD") + assert not outside_path.exists() + + def test_remote_reference_delete_cleanup_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + git_dir = base_dir / "repo" / ".git" + git_dir.mkdir(parents=True) + outside_path = base_dir / "outside_remote_delete.txt" + outside_path.write_text("do not delete\n", encoding="utf-8") + + class GitStub: + branch_called = False + + def branch(self, *args): + self.branch_called = True + + class RepoStub: + pass + + repo = RepoStub() + repo.git = GitStub() + repo.common_dir = str(git_dir) + repo.git_dir = str(git_dir) + ref = RemoteReference(repo, "../../outside_remote_delete.txt", check_path=False) + + self.assertRaises(ValueError, RemoteReference.delete, repo, ref) + assert not repo.git.branch_called + assert outside_path.read_text(encoding="utf-8") == "do not delete\n" + def test_validity_ref_names(self): + """Ensure ref names are checked for validity. + + This is based on the rules specified in: + https://git-scm.com/docs/git-check-ref-format/#_description + """ check_ref = SymbolicReference._check_ref_name_valid - # Based on the rules specified in https://git-scm.com/docs/git-check-ref-format/#_description. + # Rule 1 self.assertRaises(ValueError, check_ref, ".ref/begins/with/dot") self.assertRaises(ValueError, check_ref, "ref/component/.begins/with/dot") self.assertRaises(ValueError, check_ref, "ref/ends/with/a.lock") self.assertRaises(ValueError, check_ref, "ref/component/ends.lock/with/period_lock") + # Rule 2 check_ref("valid_one_level_refname") + # Rule 3 self.assertRaises(ValueError, check_ref, "ref/contains/../double/period") + # Rule 4 for c in " ~^:": self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{c}/character") for code in range(0, 32): self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{chr(code)}/ASCII/control_character") self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{chr(127)}/ASCII/control_character") + # Rule 5 for c in "*?[": self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{c}/character") + # Rule 6 self.assertRaises(ValueError, check_ref, "/ref/begins/with/slash") self.assertRaises(ValueError, check_ref, "ref/ends/with/slash/") self.assertRaises(ValueError, check_ref, "ref/contains//double/slash/") + # Rule 7 self.assertRaises(ValueError, check_ref, "ref/ends/with/dot.") + # Rule 8 self.assertRaises(ValueError, check_ref, "ref/contains@{/at_brace") + # Rule 9 self.assertRaises(ValueError, check_ref, "@") + # Rule 10 self.assertRaises(ValueError, check_ref, "ref/contain\\s/backslash") + # Valid reference name should not raise. check_ref("valid/ref/name") diff --git a/test/test_remote.py b/test/test_remote.py index df6034326..e1793214c 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -8,8 +8,9 @@ import os.path as osp from pathlib import Path import random +import sys import tempfile -from unittest import skipIf +from unittest import mock, skipIf import pytest @@ -28,7 +29,7 @@ ) from git.cmd import Git from git.exc import UnsafeOptionError, UnsafeProtocolError -from git.util import rmtree, HIDE_WINDOWS_FREEZE_ERRORS, IterableList +from git.util import HIDE_WINDOWS_FREEZE_ERRORS, IterableList, rmtree from test.lib import ( GIT_DAEMON_PORT, TestBase, @@ -37,7 +38,6 @@ with_rw_repo, ) - # Make sure we have repeatable results. random.seed(0) @@ -45,7 +45,7 @@ class TestRemoteProgress(RemoteProgress): __slots__ = ("_seen_lines", "_stages_per_op", "_num_progress_messages") - def __init__(self): + def __init__(self) -> None: super().__init__() self._seen_lines = [] self._stages_per_op = {} @@ -104,6 +104,9 @@ def assert_received_message(self): assert self._num_progress_messages +TestRemoteProgress.__test__ = False # type: ignore + + class TestRemote(TestBase): def tearDown(self): gc.collect() @@ -294,11 +297,11 @@ def get_info(res, remote, name): # Provoke to receive actual objects to see what kind of output we have to # expect. For that we need a remote transport protocol. - # Create a new UN-shared repo and fetch into it after we pushed a change - # to the shared repo. + # Create a new UN-shared repo and fetch into it after we pushed a change to the + # shared repo. other_repo_dir = tempfile.mktemp("other_repo") - # Must clone with a local path for the repo implementation not to freak out - # as it wants local paths only (which I can understand). + # Must clone with a local path for the repo implementation not to freak out as + # it wants local paths only (which I can understand). other_repo = remote_repo.clone(other_repo_dir, shared=False) remote_repo_url = osp.basename(remote_repo.git_dir) # git-daemon runs with appropriate `--base-path`. remote_repo_url = Git.polish_url("git://localhost:%s/%s" % (GIT_DAEMON_PORT, remote_repo_url)) @@ -317,10 +320,10 @@ def get_info(res, remote, name): self._commit_random_file(rw_repo) remote.push(rw_repo.head.reference) - # Here I would expect to see remote-information about packing - # objects and so on. Unfortunately, this does not happen - # if we are redirecting the output - git explicitly checks for this - # and only provides progress information to ttys. + # Here I would expect to see remote-information about packing objects and so + # on. Unfortunately, this does not happen if we are redirecting the output - + # git explicitly checks for this and only provides progress information to + # ttys. res = fetch_and_test(other_origin) finally: rmtree(other_repo_dir) @@ -333,8 +336,8 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo): try: lhead.reference = rw_repo.heads.master except AttributeError: - # If the author is on a non-master branch, the clones might not have - # a local master yet. We simply create it. + # If the author is on a non-master branch, the clones might not have a local + # master yet. We simply create it. lhead.reference = rw_repo.create_head("master") # END master handling lhead.reset(remote.refs.master, working_tree=True) @@ -488,8 +491,8 @@ def test_base(self, rw_repo, remote_repo): self._assert_push_and_pull(remote, rw_repo, remote_repo) # FETCH TESTING - # Only for remotes - local cases are the same or less complicated - # as additional progress information will never be emitted. + # Only for remotes - local cases are the same or less complicated as + # additional progress information will never be emitted. if remote.name == "daemon_origin": self._do_test_fetch(remote, rw_repo, remote_repo, kill_after_timeout=10.0) ran_fetch_test = True @@ -508,7 +511,8 @@ def test_base(self, rw_repo, remote_repo): # Verify we can handle prunes when fetching. # stderr lines look like this: x [deleted] (none) -> origin/experiment-2012 # These should just be skipped. - # If we don't have a manual checkout, we can't actually assume there are any non-master branches. + # If we don't have a manual checkout, we can't actually assume there are any + # non-master branches. remote_repo.create_head("myone_for_deletion") # Get the branch - to be pruned later origin.fetch() @@ -682,9 +686,22 @@ def test_multiple_urls(self, rw_repo): # Will raise fatal: Will not delete all non-push URLs. self.assertRaises(GitCommandError, remote.delete_url, test3) + @with_rw_repo("HEAD", bare=False) + def test_create_does_not_expand_environment_variables_in_url(self, rw_repo): + url = "https://example.com/${GITPYTHON_TEST_SECRET}/repo.git" + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + remote = rw_repo.create_remote("untrusted", url) + + assert remote.url == url + def test_fetch_error(self): rem = self.rorepo.remote("origin") - with self.assertRaisesRegex(GitCommandError, "[Cc]ouldn't find remote ref __BAD_REF__"): + msg = ( + r"[Cc]ouldn't find remote ref __BAD_REF__|" + r"could not read Username|" + r"expected flush after ref listing" + ) + with self.assertRaisesRegex(GitCommandError, msg): rem.fetch("__BAD_REF__") @with_rw_repo("0.1.6", bare=False) @@ -768,7 +785,7 @@ def test_create_remote_unsafe_url(self, rw_repo): assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=R"Multiple '\' instead of '/' in remote.url make it differ from expected value", raises=AssertionError, ) @@ -812,8 +829,8 @@ def test_fetch_unsafe_url_allowed(self, rw_repo): "fd::17/foo", ] for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): remote.fetch(url, allow_unsafe_protocols=True) assert not tmp_file.exists() @@ -824,14 +841,18 @@ def test_fetch_unsafe_options(self, rw_repo): remote = rw_repo.remote("origin") tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" - unsafe_options = [{"upload-pack": f"touch {tmp_file}"}] + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, + {"upload_p": f"touch {tmp_file}"}, + ] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): remote.fetch(**unsafe_option) assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=( "File not created. A separate Windows command may be needed. This and the " "currently passing test test_fetch_unsafe_options must be adjusted in the " @@ -880,8 +901,8 @@ def test_pull_unsafe_url_allowed(self, rw_repo): "fd::17/foo", ] for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): remote.pull(url, allow_unsafe_protocols=True) assert not tmp_file.exists() @@ -892,14 +913,18 @@ def test_pull_unsafe_options(self, rw_repo): remote = rw_repo.remote("origin") tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" - unsafe_options = [{"upload-pack": f"touch {tmp_file}"}] + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, + {"upload_p": f"touch {tmp_file}"}, + ] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): remote.pull(**unsafe_option) assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=( "File not created. A separate Windows command may be needed. This and the " "currently passing test test_pull_unsafe_options must be adjusted in the " @@ -948,8 +973,8 @@ def test_push_unsafe_url_allowed(self, rw_repo): "fd::17/foo", ] for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): remote.push(url, allow_unsafe_protocols=True) assert not tmp_file.exists() @@ -961,10 +986,11 @@ def test_push_unsafe_options(self, rw_repo): tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" unsafe_options = [ - { - "receive-pack": f"touch {tmp_file}", - "exec": f"touch {tmp_file}", - } + {"receive-pack": f"touch {tmp_file}"}, + {"receive_pack": f"touch {tmp_file}"}, + {"receive_p": f"touch {tmp_file}"}, + {"exec": f"touch {tmp_file}"}, + {"exe": f"touch {tmp_file}"}, ] for unsafe_option in unsafe_options: assert not tmp_file.exists() @@ -973,7 +999,7 @@ def test_push_unsafe_options(self, rw_repo): assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=( "File not created. A separate Windows command may be needed. This and the " "currently passing test test_push_unsafe_options must be adjusted in the " @@ -988,10 +1014,9 @@ def test_push_unsafe_options_allowed(self, rw_repo): tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" unsafe_options = [ - { - "receive-pack": f"touch {tmp_file}", - "exec": f"touch {tmp_file}", - } + {"receive-pack": f"touch {tmp_file}"}, + {"receive_pack": f"touch {tmp_file}"}, + {"exec": f"touch {tmp_file}"}, ] for unsafe_option in unsafe_options: # The options will be allowed, but the command will fail. @@ -1001,6 +1026,66 @@ def test_push_unsafe_options_allowed(self, rw_repo): assert tmp_file.exists() tmp_file.unlink() + @with_rw_repo("HEAD") + def test_ls_remote_unsafe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, + {"upl": f"touch {tmp_file}"}, + {"exec": f"touch {tmp_file}"}, + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote(".", **unsafe_option) + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote([f"--upload-pack={tmp_file}"], ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote([f"--upl={tmp_file}"], ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote(f"--upload-pack={tmp_file}", ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote(f"--upl={tmp_file}", ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote(f"--exec={tmp_file}", ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote("--upload-pack", "touch", ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote("--refs", ".", upl=f"touch {tmp_file}") + + def test_ls_remote_allows_operand_named_like_unsafe_option(self): + with mock.patch.object(Git, "_call_process") as git: + Git().ls_remote("upload-pack") + git.assert_called_once() + + @with_rw_repo("HEAD") + def test_ls_remote_unsafe_options_allowed(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [{"upload-pack": f"touch {tmp_file}"}] + for unsafe_option in unsafe_options: + with self.assertRaises(GitCommandError): + rw_repo.git.ls_remote(".", **unsafe_option, allow_unsafe_options=True) + + @with_rw_and_rw_remote_repo("0.1.6") + def test_fetch_unsafe_branch_name(self, rw_repo, remote_repo): + # Create branch with a name containing a NBSP + bad_branch_name = f"branch_with_{chr(160)}_nbsp" + Head.create(remote_repo, bad_branch_name) + + # Fetch and get branches + remote = rw_repo.remote("origin") + branches = remote.fetch() + + # Test for truncated branch name in branches + assert f"origin/{bad_branch_name}" in [b.name for b in branches] + + # Cleanup branch + Head.delete(remote_repo, bad_branch_name) + class TestTimeouts(TestBase): @with_rw_repo("HEAD", bare=False) diff --git a/test/test_repo.py b/test/test_repo.py index 465bb2574..1dfec951a 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -14,7 +14,8 @@ import pickle import sys import tempfile -from unittest import mock, skip +from unittest import mock +from pathlib import Path import pytest @@ -36,14 +37,13 @@ Submodule, Tree, ) -from git.exc import ( - BadObject, - UnsafeOptionError, - UnsafeProtocolError, -) +from git.exc import UnsafeOptionError +from git.exc import UnsafeProtocolError +from git.exc import BadObject from git.repo.fun import touch from git.util import bin_to_hex, cwd, cygpath, join_path_native, rmfile, rmtree -from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo + +from test.lib import TestBase, fixture, requires_symlinks, with_rw_directory, with_rw_repo, PathLikeMock def iter_flatten(lol): @@ -82,6 +82,33 @@ def test_new_should_raise_on_invalid_repo_location(self): with tempfile.TemporaryDirectory() as tdir: self.assertRaises(InvalidGitRepositoryError, Repo, tdir) + def test_init_rejects_unsafe_options(self): + with tempfile.TemporaryDirectory() as tdir: + template_dir = osp.join(tdir, "template") + os.mkdir(template_dir) + unsafe_options = [ + {"template": template_dir}, + {"templa": template_dir}, + {"separate_git_dir": osp.join(tdir, "git-dir")}, + {"separate_git_di": osp.join(tdir, "git-dir")}, + ] + for index, kwargs in enumerate(unsafe_options): + repo_dir = osp.join(tdir, f"repo-{index}") + with self.assertRaises(UnsafeOptionError): + Repo.init(repo_dir, **kwargs) + assert not osp.exists(repo_dir) + + def test_init_allows_explicitly_unsafe_options(self): + with tempfile.TemporaryDirectory() as tdir: + template_dir = osp.join(tdir, "template") + os.mkdir(template_dir) + repo = Repo.init( + osp.join(tdir, "repo"), + template=template_dir, + allow_unsafe_options=True, + ) + assert repo.git_dir + @with_rw_directory def test_new_should_raise_on_invalid_repo_location_within_repo(self, rw_dir): repo_dir = osp.join(rw_dir, "repo") @@ -108,6 +135,11 @@ def test_repo_creation_pathlib(self, rw_repo): r_from_gitdir = Repo(pathlib.Path(rw_repo.git_dir)) self.assertEqual(r_from_gitdir.git_dir, rw_repo.git_dir) + @with_rw_repo("0.3.2.1") + def test_repo_creation_pathlike(self, rw_repo): + r_from_gitdir = Repo(PathLikeMock(rw_repo.git_dir)) + self.assertEqual(r_from_gitdir.git_dir, rw_repo.git_dir) + def test_description(self): txt = "Test repository" self.rorepo.description = txt @@ -123,8 +155,9 @@ def test_heads_should_populate_head_data(self): self.assertIsInstance(head.commit, Commit) # END for each head - self.assertIsInstance(self.rorepo.heads.master, Head) - self.assertIsInstance(self.rorepo.heads["master"], Head) + active_branch = self.rorepo.active_branch + self.assertIsInstance(self.rorepo.heads[active_branch.name], Head) + self.assertEqual(self.rorepo.heads[active_branch.name], active_branch) def test_tree_from_revision(self): tree = self.rorepo.tree("0.1.6") @@ -143,6 +176,23 @@ def test_commit_from_revision(self): self.assertEqual(commit.type, "commit") self.assertEqual(self.rorepo.commit(commit), commit) + @with_rw_directory + def test_commit_from_tag_starting_with_at(self, rw_dir): + repo = Repo.init(rw_dir) + with repo.config_writer() as writer: + writer.set_value("user", "name", "GitPython Tests") + writer.set_value("user", "email", "gitpython@example.com") + + tracked_file = Path(rw_dir) / "hello.txt" + tracked_file.write_text("hello") + repo.index.add([str(tracked_file)]) + commit = repo.index.commit("init") + repo.create_tag("@foo") + + self.assertEqual(repo.tags["@foo"].commit, commit) + self.assertEqual(repo.commit("@"), commit) + self.assertEqual(repo.commit("@foo"), commit) + def test_commits(self): mc = 10 commits = list(self.rorepo.iter_commits("0.1.6", max_count=mc)) @@ -211,290 +261,23 @@ def test_clone_from_keeps_env(self, rw_dir): self.assertEqual(environment, cloned.git.environment()) + @pytest.mark.skipif(os.name != "nt", reason="Specifically for Windows drive-rooted paths.") @with_rw_directory - def test_date_format(self, rw_dir): - repo = Repo.init(osp.join(rw_dir, "repo")) - # @-timestamp is the format used by git commit hooks. - repo.index.commit("Commit messages", commit_date="@1400000000 +0000") - - @with_rw_directory - def test_clone_from_pathlib(self, rw_dir): - original_repo = Repo.init(osp.join(rw_dir, "repo")) - - Repo.clone_from(original_repo.git_dir, pathlib.Path(rw_dir) / "clone_pathlib") - - @with_rw_directory - def test_clone_from_pathlib_withConfig(self, rw_dir): + def test_clone_from_drive_rooted_destination(self, rw_dir): original_repo = Repo.init(osp.join(rw_dir, "repo")) + with cwd(rw_dir): + destination = osp.join(rw_dir, "clone") + _drive, rooted_destination = osp.splitdrive(destination) - cloned = Repo.clone_from( - original_repo.git_dir, - pathlib.Path(rw_dir) / "clone_pathlib_withConfig", - multi_options=[ - "--recurse-submodules=repo", - "--config core.filemode=false", - "--config submodule.repo.update=checkout", - "--config filter.lfs.clean='git-lfs clean -- %f'", - ], - allow_unsafe_options=True, - ) - - self.assertEqual(cloned.config_reader().get_value("submodule", "active"), "repo") - self.assertEqual(cloned.config_reader().get_value("core", "filemode"), False) - self.assertEqual(cloned.config_reader().get_value('submodule "repo"', "update"), "checkout") - self.assertEqual( - cloned.config_reader().get_value('filter "lfs"', "clean"), - "git-lfs clean -- %f", - ) - - def test_clone_from_with_path_contains_unicode(self): - with tempfile.TemporaryDirectory() as tmpdir: - unicode_dir_name = "\u0394" - path_with_unicode = os.path.join(tmpdir, unicode_dir_name) - os.makedirs(path_with_unicode) + cloned = Repo.clone_from(original_repo.git_dir, rooted_destination) - try: - Repo.clone_from( - url=self._small_repo_url(), - to_path=path_with_unicode, - ) - except UnicodeEncodeError: - self.fail("Raised UnicodeEncodeError") + assert osp.samefile(cloned.working_tree_dir, destination) @with_rw_directory - @skip( - """The referenced repository was removed, and one needs to set up a new - password controlled repo under the org's control.""" - ) - def test_leaking_password_in_clone_logs(self, rw_dir): - password = "fakepassword1234" - try: - Repo.clone_from( - url="https://fakeuser:{}@fakerepo.example.com/testrepo".format(password), - to_path=rw_dir, - ) - except GitCommandError as err: - assert password not in str(err), "The error message '%s' should not contain the password" % err - # Working example from a blank private project. - Repo.clone_from( - url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", - to_path=rw_dir, - ) - - @with_rw_repo("HEAD") - def test_clone_unsafe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) - assert not tmp_file.exists() - - unsafe_options = [ - {"upload-pack": f"touch {tmp_file}"}, - {"u": f"touch {tmp_file}"}, - {"config": "protocol.ext.allow=always"}, - {"c": "protocol.ext.allow=always"}, - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - rw_repo.clone(tmp_dir, **unsafe_option) - assert not tmp_file.exists() - - @pytest.mark.xfail( - os.name == "nt", - reason=( - "File not created. A separate Windows command may be needed. This and the " - "currently passing test test_clone_unsafe_options must be adjusted in the " - "same way. Until then, test_clone_unsafe_options is unreliable on Windows." - ), - raises=AssertionError, - ) - @with_rw_repo("HEAD") - def test_clone_unsafe_options_allowed(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not tmp_file.exists() - # The options will be allowed, but the command will fail. - with self.assertRaises(GitCommandError): - rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) - assert tmp_file.exists() - tmp_file.unlink() - - unsafe_options = [ - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not destination.exists() - rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_safe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - options = [ - "--depth=1", - "--single-branch", - "-q", - ] - for option in options: - destination = tmp_dir / option - assert not destination.exists() - rw_repo.clone(destination, multi_options=[option]) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_from_unsafe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - Repo.clone_from(rw_repo.working_dir, tmp_dir, multi_options=[unsafe_option]) - assert not tmp_file.exists() - - unsafe_options = [ - {"upload-pack": f"touch {tmp_file}"}, - {"u": f"touch {tmp_file}"}, - {"config": "protocol.ext.allow=always"}, - {"c": "protocol.ext.allow=always"}, - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - Repo.clone_from(rw_repo.working_dir, tmp_dir, **unsafe_option) - assert not tmp_file.exists() - - @pytest.mark.xfail( - os.name == "nt", - reason=( - "File not created. A separate Windows command may be needed. This and the " - "currently passing test test_clone_from_unsafe_options must be adjusted in the " - "same way. Until then, test_clone_from_unsafe_options is unreliable on Windows." - ), - raises=AssertionError, - ) - @with_rw_repo("HEAD") - def test_clone_from_unsafe_options_allowed(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not tmp_file.exists() - # The options will be allowed, but the command will fail. - with self.assertRaises(GitCommandError): - Repo.clone_from( - rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True - ) - assert tmp_file.exists() - tmp_file.unlink() - - unsafe_options = [ - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not destination.exists() - Repo.clone_from( - rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True - ) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_from_safe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - options = [ - "--depth=1", - "--single-branch", - "-q", - ] - for option in options: - destination = tmp_dir / option - assert not destination.exists() - Repo.clone_from(rw_repo.common_dir, destination, multi_options=[option]) - assert destination.exists() - - def test_clone_from_unsafe_protocol(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - "fd::17/foo", - ] - for url in urls: - with self.assertRaises(UnsafeProtocolError): - Repo.clone_from(url, tmp_dir / "repo") - assert not tmp_file.exists() - - def test_clone_from_unsafe_protocol_allowed(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - "fd::/foo", - ] - for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. - with self.assertRaises(GitCommandError): - Repo.clone_from(url, tmp_dir / "repo", allow_unsafe_protocols=True) - assert not tmp_file.exists() - - def test_clone_from_unsafe_protocol_allowed_and_enabled(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - ] - allow_ext = [ - "--config=protocol.ext.allow=always", - ] - for url in urls: - # The URL will be allowed into the command, and the protocol is enabled, - # but the command will fail since it can't read from the remote repo. - assert not tmp_file.exists() - with self.assertRaises(GitCommandError): - Repo.clone_from( - url, - tmp_dir / "repo", - multi_options=allow_ext, - allow_unsafe_protocols=True, - allow_unsafe_options=True, - ) - assert tmp_file.exists() - tmp_file.unlink() + def test_date_format(self, rw_dir): + repo = Repo.init(osp.join(rw_dir, "repo")) + # @-timestamp is the format used by git commit hooks. + repo.index.commit("Commit messages", commit_date="@1400000000 +0000") @with_rw_repo("HEAD") def test_max_chunk_size(self, repo): @@ -543,8 +326,8 @@ def test_init(self): try: rmtree(clone_path) except OSError: - # When relative paths are used, the clone may actually be inside - # of the parent directory. + # When relative paths are used, the clone may actually be inside of + # the parent directory. pass # END exception handling @@ -556,8 +339,8 @@ def test_init(self): try: rmtree(clone_path) except OSError: - # When relative paths are used, the clone may actually be inside - # of the parent directory. + # When relative paths are used, the clone may actually be inside of + # the parent directory. pass # END exception handling @@ -581,15 +364,37 @@ def test_daemon_export(self): self.rorepo.daemon_export = orig_val self.assertEqual(self.rorepo.daemon_export, orig_val) + def test_daemon_export_without_git_dir(self): + repo = Repo.__new__(Repo) + repo.git_dir = None + self.assertFalse(repo._get_daemon_export()) + repo._set_daemon_export(True) + def test_alternates(self): cur_alternates = self.rorepo.alternates - # empty alternates - self.rorepo.alternates = [] - self.assertEqual(self.rorepo.alternates, []) + try: + # Empty alternates. + self.rorepo.alternates = [] + self.assertEqual(self.rorepo.alternates, []) + alts = ["other/location", "this/location"] + self.rorepo.alternates = alts + self.assertEqual(alts, self.rorepo.alternates) + finally: + self.rorepo.alternates = cur_alternates + + @with_rw_directory + def test_alternates_use_common_dir(self, rw_dir): + common_dir = osp.join(rw_dir, "common") + git_dir = osp.join(rw_dir, "worktrees", "linked") + os.makedirs(osp.join(common_dir, "objects", "info")) + os.makedirs(osp.join(git_dir, "objects", "info")) + repo = mock.Mock(common_dir=common_dir, git_dir=git_dir) + alts = ["other/location", "this/location"] - self.rorepo.alternates = alts - self.assertEqual(alts, self.rorepo.alternates) - self.rorepo.alternates = cur_alternates + Repo._set_alternates(repo, alts) + + self.assertEqual(Repo._get_alternates(repo), alts) + self.assertFalse(osp.exists(osp.join(git_dir, "objects", "info", "alternates"))) def test_repr(self): assert repr(self.rorepo).startswith("= (2, 48, 0): + wt_add_kwargs["relative_paths"] = True + rw_master.git.worktree("add", worktree_path, branch.name, **wt_add_kwargs) # This ensures that we can read the repo's gitdir correctly. repo = Repo(worktree_path) self.assertIsInstance(repo, Repo) - # This ensures we're able to actually read the refs in the tree, which - # means we can read commondir correctly. + # This ensures we're able to actually read the refs in the tree, which means we + # can read commondir correctly. commit = repo.head.commit self.assertIsInstance(commit, Object) - # This ensures we can read the remotes, which confirms we're reading - # the config correctly. + # This ensures we can read the remotes, which confirms we're reading the config + # correctly. origin = repo.remotes.origin self.assertIsInstance(origin, Remote) self.assertIsInstance(repo.heads["aaaaaaaa"], Head) + def test_git_work_tree_dotgit_relative(self): + """Check that we find .git as a worktree file containing a relative path + and find the worktree based on it.""" + if Git().version_info[:3] < (2, 48, 0): + pytest.skip("relative worktree feature unsupported, needs git 2.48.0 or later") + # this class inherits from TestCase so we can't use pytest.mark.parametrize on + # test_git_work_tree_dotgit; delegate instead + self.test_git_work_tree_dotgit(use_relative_paths=True) + + @pytest.mark.skipif(os.name != "nt", reason="Specifically for Windows drive-rooted paths.") + @with_rw_directory + def test_linked_worktree_drive_rooted_gitdir_file(self, rw_dir): + with cwd(rw_dir): + main_repo = self.rorepo.clone(join_path_native(rw_dir, "main_repo")) + branch = main_repo.create_head("drive-rooted-worktree") + worktree_path = join_path_native(rw_dir, "worktree_repo") + main_repo.git.worktree("add", worktree_path, branch.name) + + linked_repo = Repo(worktree_path) + gitdir_file = osp.join(linked_repo.git_dir, "gitdir") + worktree_gitfile = osp.join(worktree_path, ".git") + _drive, rooted_worktree_gitfile = osp.splitdrive(worktree_gitfile) + with open(gitdir_file, "w") as stream: + stream.write(rooted_worktree_gitfile) + + reopened_repo = Repo(linked_repo.git_dir) + + assert osp.samefile(reopened_repo.working_tree_dir, worktree_path) + @with_rw_directory def test_git_work_tree_env(self, rw_dir): """Check that we yield to GIT_WORK_TREE.""" @@ -1351,6 +1314,42 @@ def test_git_work_tree_env(self, rw_dir): self.assertEqual(r.working_tree_dir, repo_dir) self.assertEqual(r.working_dir, repo_dir) + @with_rw_directory + def test_git_work_tree_env_in_linked_worktree(self, rw_dir): + """Check that Repo() autodiscovers a linked worktree when GIT_DIR is set.""" + git = Git(rw_dir) + if git.version_info[:3] < (2, 5, 1): + raise RuntimeError("worktree feature unsupported (test needs git 2.5.1 or later)") + + rw_master = self.rorepo.clone(join_path_native(rw_dir, "master_repo")) + branch = rw_master.create_head("bbbbbbbb") + worktree_path = join_path_native(rw_dir, "worktree_repo") + if Git.is_cygwin(): + worktree_path = cygpath(worktree_path) + + rw_master.git.worktree("add", worktree_path, branch.name) + + git_dir = Git(worktree_path).rev_parse("--git-dir") + + patched_env = dict(os.environ) + patched_env["GIT_DIR"] = git_dir + patched_env.pop("GIT_WORK_TREE", None) + patched_env.pop("GIT_COMMON_DIR", None) + + with mock.patch.dict(os.environ, patched_env, clear=True): + old_cwd = os.getcwd() + try: + os.chdir(worktree_path) + + explicit = Repo(os.getcwd()) + autodiscovered = Repo() + + self.assertTrue(osp.samefile(explicit.working_tree_dir, worktree_path)) + self.assertTrue(osp.samefile(autodiscovered.working_tree_dir, worktree_path)) + self.assertTrue(osp.samefile(autodiscovered.working_tree_dir, explicit.working_tree_dir)) + finally: + os.chdir(old_cwd) + @with_rw_directory def test_rebasing(self, rw_dir): r = Repo.init(rw_dir) @@ -1387,7 +1386,7 @@ def test_do_not_strip_newline_in_stdout(self, rw_dir): self.assertEqual(r.git.show("HEAD:hello.txt", strip_newline_in_stdout=False), "hello\n") @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=R"fatal: could not create leading directories of '--upload-pack=touch C:\Users\ek\AppData\Local\Temp\tmpnantqizc\pwn': Invalid argument", # noqa: E501 raises=GitCommandError, ) @@ -1437,6 +1436,7 @@ def test_ignored_items_reported(self): ["included_file.txt", "ignored_file.txt", "included_dir/file.txt", "ignored_dir/file.txt"] ) == ["ignored_file.txt", "ignored_dir/file.txt"] + @requires_symlinks def test_ignored_raises_error_w_symlink(self): with tempfile.TemporaryDirectory() as tdir: tmp_dir = pathlib.Path(tdir) diff --git a/test/test_rev_parse.py b/test/test_rev_parse.py new file mode 100644 index 000000000..b00347668 --- /dev/null +++ b/test/test_rev_parse.py @@ -0,0 +1,159 @@ +# Copyright (C) 2026 Michael Trier (mtrier@gmail.com) and contributors +# +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +from pathlib import Path + +import pytest + +from git import Repo +from git.refs import RemoteReference +from git.refs import SymbolicReference +from gitdb.exc import BadName + + +def _write(repo, path, content): + full_path = Path(repo.working_tree_dir) / path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_text(content) + repo.index.add([str(full_path)]) + + +@pytest.fixture +def rev_parse_repo(tmp_path): + repo = Repo.init(tmp_path) + with repo.config_writer() as writer: + writer.set_value("user", "name", "GitPython Tests") + writer.set_value("user", "email", "gitpython@example.com") + + _write(repo, "README.md", "root\n") + _write(repo, "CHANGES", "root changes\n") + _write(repo, "dir/file.txt", "root file\n") + root = repo.index.commit("root commit") + repo.create_tag("ann", ref=root, message="annotated tag") + + _write(repo, "README.md", "release\n") + release = repo.index.commit("release candidate") + repo.create_tag("v1.0", ref=release) + main = repo.active_branch + + _write(repo, "side.txt", "side\n") + side_commit = repo.index.commit("side branch", parent_commits=[root], head=False, skip_hooks=True) + repo.create_head("side", side_commit) + + merge = repo.index.commit("merge side", parent_commits=[release, side_commit], skip_hooks=True) + repo.head.log_append(side_commit.binsha, "checkout: moving from side to main", merge.binsha) + + repo.create_head("aaaaaaaa", merge) + repo.create_tag("@foo", ref=merge) + + return { + "repo": repo, + "root": root, + "release": release, + "side": side_commit, + "merge": merge, + "main": main, + } + + +def test_rev_parse_names_hex_and_describe_forms(rev_parse_repo): + repo = rev_parse_repo["repo"] + release = rev_parse_repo["release"] + merge = rev_parse_repo["merge"] + + assert repo.rev_parse("@") == merge + assert repo.rev_parse("@foo") == merge + assert repo.rev_parse("aaaaaaaa") == merge + assert repo.rev_parse(merge.hexsha[:7]) == merge + describe_name = "anything-9-g%s" % merge.hexsha[:7] + assert repo.rev_parse("v1.0-1-g%s" % merge.hexsha[:7]) == merge + assert repo.rev_parse(describe_name) == merge + assert repo.rev_parse("%s-dirty" % merge.hexsha[:7]) == merge + + repo.create_tag(describe_name, ref=release) + assert repo.rev_parse(describe_name) == release + + +def test_rev_parse_navigation_and_peeling(rev_parse_repo): + repo = rev_parse_repo["repo"] + root = rev_parse_repo["root"] + release = rev_parse_repo["release"] + side = rev_parse_repo["side"] + merge = rev_parse_repo["merge"] + tag = repo.rev_parse("ann") + + assert repo.rev_parse("HEAD^0") == merge + assert repo.rev_parse("HEAD~0") == merge + assert repo.rev_parse("HEAD^1") == release + assert repo.rev_parse("HEAD^2") == side + assert repo.rev_parse("HEAD~") == release + assert repo.rev_parse("HEAD^^") == root + + assert tag.type == "tag" + assert repo.rev_parse("ann^{object}") == tag + assert repo.rev_parse("ann^{tag}") == tag + assert repo.rev_parse("ann^{}") == root + assert repo.rev_parse("ann^{commit}") == root + assert repo.rev_parse("HEAD^{tree}") == merge.tree + with pytest.raises(ValueError): + repo.rev_parse("HEAD^{/}") + + +def test_rev_parse_tree_and_index_paths(rev_parse_repo): + repo = rev_parse_repo["repo"] + merge = rev_parse_repo["merge"] + + assert repo.rev_parse("HEAD:") == merge.tree + assert repo.rev_parse("HEAD:README.md") == merge.tree["README.md"] + assert repo.rev_parse("HEAD^{tree}:README.md") == merge.tree["README.md"] + assert repo.rev_parse(":README.md").binsha == merge.tree["README.md"].binsha + assert repo.rev_parse(":0:README.md").binsha == merge.tree["README.md"].binsha + + +def test_rev_parse_reflog_selectors(rev_parse_repo): + repo = rev_parse_repo["repo"] + merge = rev_parse_repo["merge"] + side = rev_parse_repo["side"] + main = rev_parse_repo["main"] + release = rev_parse_repo["release"] + + assert repo.rev_parse("@{0}") == merge + assert repo.rev_parse("@{+0}") == merge + assert repo.rev_parse("@{1}") == release + assert repo.rev_parse("%s@{0}" % main.name) == merge + assert repo.rev_parse("@{-1}") == side + + SymbolicReference.create(repo, "refs/remotes/origin/%s" % main.name, merge) + main.set_tracking_branch(RemoteReference(repo, "refs/remotes/origin/%s" % main.name)) + assert repo.rev_parse("%s@{upstream}" % main.name) == merge + + +def test_rev_parse_commit_message_search(rev_parse_repo): + repo = rev_parse_repo["repo"] + release = rev_parse_repo["release"] + merge = rev_parse_repo["merge"] + + assert repo.rev_parse(":/release") == release + assert repo.rev_parse("HEAD^{/release}") == release + assert repo.rev_parse("HEAD^{/!-release}") == merge + + +def test_rev_parse_rejects_invalid_object_specs(rev_parse_repo): + repo = rev_parse_repo["repo"] + + with pytest.raises(ValueError): + repo.rev_parse(":") + with pytest.raises(ValueError): + repo.rev_parse(":/") + with pytest.raises(ValueError): + repo.rev_parse(":/[") + with pytest.raises(ValueError): + repo.rev_parse("HEAD^{/[}") + with pytest.raises(ValueError): + repo.rev_parse("@{-0}") + with pytest.raises(ValueError): + repo.rev_parse("HEAD^{invalid}") + with pytest.raises(BadName): + repo.rev_parse(":missing") diff --git a/test/test_stats.py b/test/test_stats.py index 4efb6f313..91d2cf6ae 100644 --- a/test/test_stats.py +++ b/test/test_stats.py @@ -3,23 +3,30 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from test.lib import TestBase, fixture from git import Stats from git.compat import defenc +from test.lib import TestBase, fixture + class TestStats(TestBase): def test_list_from_string(self): output = fixture("diff_numstat").decode(defenc) stats = Stats._list_from_string(self.rorepo, output) - self.assertEqual(2, stats.total["files"]) - self.assertEqual(52, stats.total["lines"]) - self.assertEqual(29, stats.total["insertions"]) + self.assertEqual(3, stats.total["files"]) + self.assertEqual(59, stats.total["lines"]) + self.assertEqual(36, stats.total["insertions"]) self.assertEqual(23, stats.total["deletions"]) self.assertEqual(29, stats.files["a.txt"]["insertions"]) self.assertEqual(18, stats.files["a.txt"]["deletions"]) + self.assertEqual("M", stats.files["a.txt"]["change_type"]) self.assertEqual(0, stats.files["b.txt"]["insertions"]) self.assertEqual(5, stats.files["b.txt"]["deletions"]) + self.assertEqual("M", stats.files["b.txt"]["change_type"]) + + self.assertEqual(7, stats.files["c.txt"]["insertions"]) + self.assertEqual(0, stats.files["c.txt"]["deletions"]) + self.assertEqual("A", stats.files["c.txt"]["change_type"]) diff --git a/test/test_submodule.py b/test/test_submodule.py index 993f6b57e..d545cc8d5 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -26,8 +26,9 @@ from git.objects.submodule.base import Submodule from git.objects.submodule.root import RootModule, RootUpdateProgress from git.repo.fun import find_submodule_git_dir, touch -from git.util import HIDE_WINDOWS_KNOWN_ERRORS, join_path_native, to_native_path_linux -from test.lib import TestBase, with_rw_directory, with_rw_repo +from git.util import HIDE_WINDOWS_KNOWN_ERRORS, cwd, join_path_native, to_native_path_linux + +from test.lib import TestBase, with_rw_directory, with_rw_repo, PathLikeMock @contextlib.contextmanager @@ -57,6 +58,7 @@ def update(self, op, cur_count, max_count, message=""): print(op, cur_count, max_count, message) +TestRootProgress.__test__ = False prog = TestRootProgress() @@ -94,13 +96,15 @@ def _do_base_tests(self, rwrepo): # The module is not checked-out yet. self.assertRaises(InvalidGitRepositoryError, sm.module) - # ...which is why we can't get the branch either - it points into the module() repository. + # ...which is why we can't get the branch either - it points into the module() + # repository. self.assertRaises(InvalidGitRepositoryError, getattr, sm, "branch") # branch_path works, as it's just a string. assert isinstance(sm.branch_path, str) - # Some commits earlier we still have a submodule, but it's at a different commit. + # Some commits earlier we still have a submodule, but it's at a different + # commit. smold = next(Submodule.iter_items(rwrepo, self.k_subm_changed)) assert smold.binsha != sm.binsha assert smold != sm # the name changed @@ -120,7 +124,7 @@ def _do_base_tests(self, rwrepo): else: with sm.config_writer() as writer: # For faster checkout, set the url to the local path. - new_smclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path)) + new_smclone_path = self._gitdb_repo_url() writer.set_value("url", new_smclone_path) writer.release() assert sm.config_reader().get_value("url") == new_smclone_path @@ -141,11 +145,12 @@ def _do_base_tests(self, rwrepo): smold.set_parent_commit(self.k_subm_changed + "~1") assert smold.binsha != sm.binsha - # Raises if the sm didn't exist in new parent - it keeps its - # parent_commit unchanged. + # Raises if the sm didn't exist in new parent - it keeps its parent_commit + # unchanged. self.assertRaises(ValueError, smold.set_parent_commit, self.k_no_subm_tag) - # TEST TODO: If a path is in the .gitmodules file, but not in the index, it raises. + # TODO: Test that, if a path is in the .gitmodules file, but not in the index, + # then it raises. # TEST UPDATE ############## @@ -170,6 +175,10 @@ def _do_base_tests(self, rwrepo): sma = Submodule.add(rwrepo, sm.name, sm.path) assert sma.path == sm.path + # Adding existing as pathlike + sma = Submodule.add(rwrepo, sm.name, PathLikeMock(sm.path)) + assert sma.path == sm.path + # No url and no module at path fails. self.assertRaises(ValueError, Submodule.add, rwrepo, "newsubm", "pathtorepo", url=None) @@ -196,8 +205,8 @@ def _do_base_tests(self, rwrepo): # INTERLEAVE ADD TEST ##################### - # url must match the one in the existing repository (if submodule name suggests a new one) - # or we raise. + # url must match the one in the existing repository (if submodule name + # suggests a new one) or we raise. self.assertRaises( ValueError, Submodule.add, @@ -228,8 +237,9 @@ def _do_base_tests(self, rwrepo): assert not csm.module_exists() csm_repopath = csm.path - # Adjust the path of the submodules module to point to the local destination. - new_csmclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path, csm.path)) + # Adjust the path of the submodules module to point to the local + # destination. + new_csmclone_path = self._small_repo_url() with csm.config_writer() as writer: writer.set_value("url", new_csmclone_path) assert csm.url == new_csmclone_path @@ -249,7 +259,8 @@ def _do_base_tests(self, rwrepo): # This flushed in a sub-submodule. assert len(list(rwrepo.iter_submodules())) == 2 - # Reset both heads to the previous version, verify that to_latest_revision works. + # Reset both heads to the previous version, verify that to_latest_revision + # works. smods = (sm.module(), csm.module()) for repo in smods: repo.head.reset("HEAD~2", working_tree=1) @@ -296,8 +307,8 @@ def _do_base_tests(self, rwrepo): # Must delete something. self.assertRaises(ValueError, csm.remove, module=False, configuration=False) - # module() is supposed to point to gitdb, which has a child-submodule whose URL is still pointing - # to GitHub. To save time, we will change it to: + # module() is supposed to point to gitdb, which has a child-submodule whose + # URL is still pointing to GitHub. To save time, we will change it to: csm.set_parent_commit(csm.repo.head.commit) with csm.config_writer() as cw: cw.set_value("url", self._small_repo_url()) @@ -399,8 +410,8 @@ def _do_base_tests(self, rwrepo): rwrepo.index.commit("my submod commit") assert len(rwrepo.submodules) == 2 - # Needs update, as the head changed. It thinks it's in the history - # of the repo otherwise. + # Needs update, as the head changed. + # It thinks it's in the history of the repo otherwise. nsm.set_parent_commit(rwrepo.head.commit) osm.set_parent_commit(rwrepo.head.commit) @@ -434,7 +445,8 @@ def _do_base_tests(self, rwrepo): # REMOVE 'EM ALL ################ - # If a submodule's repo has no remotes, it can't be added without an explicit url. + # If a submodule's repo has no remotes, it can't be added without an + # explicit url. osmod = osm.module() osm.remove(module=False) @@ -468,11 +480,6 @@ def test_base_rw(self, rwrepo): def test_base_bare(self, rwrepo): self._do_base_tests(rwrepo) - @pytest.mark.xfail( - sys.platform == "cygwin", - reason="Cygwin GitPython can't find submodule SHA", - raises=ValueError, - ) @pytest.mark.xfail( HIDE_WINDOWS_KNOWN_ERRORS, reason=( @@ -484,15 +491,15 @@ def test_base_bare(self, rwrepo): @with_rw_repo(k_subm_current, bare=False) def test_root_module(self, rwrepo): # Can query everything without problems. - rm = RootModule(self.rorepo) - assert rm.module() is self.rorepo + rm = RootModule(rwrepo) + assert rm.module() is rwrepo # Try attributes. rm.binsha rm.mode rm.path assert rm.name == rm.k_root_name - assert rm.parent_commit == self.rorepo.head.commit + assert rm.parent_commit == rwrepo.head.commit rm.url rm.branch @@ -501,16 +508,16 @@ def test_root_module(self, rwrepo): with rm.config_writer(): pass - # Deep traversal gitdb / async. - rsmsp = [sm.path for sm in rm.traverse()] - assert len(rsmsp) >= 2 # gitdb and async [and smmap], async being a child of gitdb. - # Cannot set the parent commit as root module's path didn't exist. self.assertRaises(ValueError, rm.set_parent_commit, "HEAD") + with mock.patch.object(RootModule, "list_items", side_effect=ValueError("boom")): + rm.update(keep_going=True) + # TEST UPDATE ############# - # Set up a commit that removes existing, adds new and modifies existing submodules. + # Set up a commit that removes existing, adds new and modifies existing + # submodules. rm = RootModule(rwrepo) assert len(rm.children()) == 1 @@ -525,7 +532,7 @@ def test_root_module(self, rwrepo): # Ensure we clone from a local source. with sm.config_writer() as writer: - writer.set_value("url", Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path))) + writer.set_value("url", self._gitdb_repo_url()) # Dry-run does nothing. sm.update(recursive=False, dry_run=True, progress=prog) @@ -534,13 +541,15 @@ def test_root_module(self, rwrepo): sm.update(recursive=False) assert sm.module_exists() with sm.config_writer() as writer: - writer.set_value("path", fp) # Change path to something with prefix AFTER url change. + # Change path to something with prefix AFTER url change. + writer.set_value("path", fp) - # Update doesn't fail, because list_items ignores the wrong path in such situations. + # Update doesn't fail, because list_items ignores the wrong path in such + # situations. rm.update(recursive=False) - # Move it properly - doesn't work as it its path currently points to an indexentry - # which doesn't exist (move it to some path, it doesn't matter here). + # Move it properly - doesn't work as it its path currently points to an + # indexentry which doesn't exist (move it to some path, it doesn't matter here). self.assertRaises(InvalidGitRepositoryError, sm.move, pp) # Reset the path(cache) to where it was, now it works. sm.path = prep @@ -558,7 +567,7 @@ def test_root_module(self, rwrepo): # ============== nsmn = "newsubmodule" nsmp = "submrepo" - subrepo_url = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0], rsmsp[1])) + subrepo_url = self._small_repo_url() nsm = Submodule.add(rwrepo, nsmn, nsmp, url=subrepo_url) csmadded = rwrepo.index.commit("Added submodule").hexsha # Make sure we don't keep the repo reference. nsm.set_parent_commit(csmadded) @@ -588,23 +597,27 @@ def test_root_module(self, rwrepo): rm.update(recursive=False, dry_run=True, force_remove=True) assert osp.isdir(smp) - # When removing submodules, we may get new commits as nested submodules are auto-committing changes - # to allow deletions without force, as the index would be dirty otherwise. + # When removing submodules, we may get new commits as nested submodules are + # auto-committing changes to allow deletions without force, as the index would + # be dirty otherwise. # QUESTION: Why does this seem to work in test_git_submodule_compatibility() ? self.assertRaises(InvalidGitRepositoryError, rm.update, recursive=False, force_remove=False) rm.update(recursive=False, force_remove=True) assert not osp.isdir(smp) - # 'Apply work' to the nested submodule and ensure this is not removed/altered during updates - # Need to commit first, otherwise submodule.update wouldn't have a reason to change the head. + # 'Apply work' to the nested submodule and ensure this is not removed/altered + # during updates. We need to commit first, otherwise submodule.update wouldn't + # have a reason to change the head. touch(osp.join(nsm.module().working_tree_dir, "new-file")) - # We cannot expect is_dirty to even run as we wouldn't reset a head to the same location. + # We cannot expect is_dirty to even run as we wouldn't reset a head to the same + # location. assert nsm.module().head.commit.hexsha == nsm.hexsha nsm.module().index.add([nsm]) nsm.module().index.commit("added new file") rm.update(recursive=False, dry_run=True, progress=prog) # Would not change head, and thus doesn't fail. - # Everything we can do from now on will trigger the 'future' check, so no is_dirty() check will even run. - # This would only run if our local branch is in the past and we have uncommitted changes. + # Everything we can do from now on will trigger the 'future' check, so no + # is_dirty() check will even run. This would only run if our local branch is in + # the past and we have uncommitted changes. prev_commit = nsm.module().head.commit rm.update(recursive=False, dry_run=False, progress=prog) @@ -616,10 +629,10 @@ def test_root_module(self, rwrepo): # Change url... # ============= - # ...to the first repository. This way we have a fast checkout, and a completely different - # repository at the different url. + # ...to the first repository. This way we have a fast checkout, and a completely + # different repository at the different url. nsm.set_parent_commit(csmremoved) - nsmurl = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0])) + nsmurl = self._gitdb_repo_url() with nsm.config_writer() as writer: writer.set_value("url", nsmurl) csmpathchange = rwrepo.index.commit("changed url") @@ -637,16 +650,15 @@ def test_root_module(self, rwrepo): assert len(rwrepo.submodules) == 1 assert not rwrepo.submodules[0].children()[0].module_exists(), "nested submodule should not be checked out" - # Add the submodule's changed commit to the index, which is what the - # user would do. - # Beforehand, update our instance's binsha with the new one. + # Add the submodule's changed commit to the index, which is what the user would + # do. Beforehand, update our instance's binsha with the new one. nsm.binsha = nsm.module().head.commit.binsha rwrepo.index.add([nsm]) # Change branch. # ============== - # We only have one branch, so we switch to a virtual one, and back - # to the current one to trigger the difference. + # We only have one branch, so we switch to a virtual one, and back to the + # current one to trigger the difference. cur_branch = nsm.branch nsmm = nsm.module() prev_commit = nsmm.head.commit @@ -700,6 +712,166 @@ def test_iter_items_from_invalid_hash(self): next(it) self.assertIsNone(ctx.exception.value) + @with_rw_directory + def test_deinit_calls_git_submodule(self, rwdir): + repo = git.Repo.init(rwdir) + submodule = Submodule(repo, b"\0" * 20, name="module", path="module") + + with mock.patch.object(Git, "submodule", create=True) as git_submodule: + submodule.deinit() + + git_submodule.assert_called_once_with("deinit", "--", submodule.path) + git_submodule.reset_mock() + + submodule.deinit(force=True) + + git_submodule.assert_called_once_with("deinit", "--force", "--", submodule.path) + + @with_rw_directory + def test_update_after_deinit(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + touch(osp.join(source_path, "file")) + source_repo.index.add(["file"]) + source_repo.index.commit("initial commit") + + parent_path = osp.join(rwdir, "parent") + parent_repo = git.Repo.init(parent_path) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + + submodule.deinit() + assert not submodule.module_exists() + assert osp.isdir(osp.join(parent_repo.git_dir, "modules", submodule.name)) + + submodule.update() + + assert submodule.module_exists() + assert submodule.module().head.commit == source_repo.head.commit + assert osp.isfile(osp.join(submodule.abspath, "file")) + + @with_rw_directory + def test_update_to_latest_revision_after_deinit_fetches_remote(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + source_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + touch(osp.join(source_path, "new-file")) + source_repo.index.add(["new-file"]) + source_repo.index.commit("advance remote") + + submodule.update(to_latest_revision=True) + + assert submodule.module().head.commit == source_repo.head.commit + assert osp.isfile(osp.join(submodule.abspath, "new-file")) + + @with_rw_directory + def test_update_after_deinit_fetches_new_gitlink_commit(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + source_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + touch(osp.join(source_path, "new-file")) + source_repo.index.add(["new-file"]) + source_repo.index.commit("advance remote") + parent_repo.git.update_index( + "--cacheinfo", + f"160000,{source_repo.head.commit.hexsha},{submodule.path}", + ) + parent_repo.index.commit("advance submodule") + + submodule = parent_repo.submodule(submodule.name) + submodule.update() + + assert submodule.module().head.commit == source_repo.head.commit + assert osp.isfile(osp.join(submodule.abspath, "new-file")) + + @with_rw_directory + def test_update_after_deinit_refuses_non_empty_checkout(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + tracked_file = osp.join(source_path, "file") + with open(tracked_file, "w") as fp: + fp.write("submodule content") + source_repo.index.add(["file"]) + source_repo.index.commit("initial commit") + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + checkout_file = osp.join(submodule.abspath, "file") + with open(checkout_file, "w") as fp: + fp.write("user content") + + with pytest.raises(OSError, match="does already exist and is non-empty"): + submodule.update() + + with open(checkout_file) as fp: + assert fp.read() == "user content" + assert not osp.exists(osp.join(submodule.abspath, ".git")) + + @with_rw_directory + def test_update_after_deinit_without_init_does_not_restore_checkout(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + source_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + assert submodule.update(init=False) is submodule + assert not submodule.module_exists() + assert not osp.exists(osp.join(submodule.abspath, ".git")) + + @with_rw_directory + def test_dry_run_update_after_deinit_does_not_restore_checkout(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + source_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + assert submodule.update(dry_run=True) is submodule + assert not submodule.module_exists() + assert not osp.exists(osp.join(submodule.abspath, ".git")) + + @with_rw_directory + def test_update_after_deinit_restores_nested_checkout(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + touch(osp.join(source_path, "file")) + source_repo.index.add(["file"]) + source_repo.index.commit("initial commit") + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("nested/module", "deps/module", source_path) + parent_repo.index.commit("add nested submodule") + submodule.deinit() + + submodule.update() + + module_repo = submodule.module() + assert module_repo.head.commit == source_repo.head.commit + assert osp.isfile(osp.join(submodule.abspath, "file")) + assert osp.samefile(module_repo.git_dir, osp.join(parent_repo.git_dir, "modules", submodule.name)) + @with_rw_repo(k_no_subm_tag, bare=False) def test_first_submodule(self, rwrepo): assert len(list(rwrepo.iter_submodules())) == 0 @@ -740,6 +912,98 @@ def test_add_empty_repo(self, rwdir): ) # END for each checkout mode + @with_rw_directory + @_patch_git_config("protocol.file.allow", "always") + def test_update_submodule_with_relative_path(self, rwdir): + repo_path = osp.join(rwdir, "parent") + repo = git.Repo.init(repo_path) + module_repo_path = osp.join(rwdir, "module") + module_repo = git.Repo.init(module_repo_path) + module_repo.git.commit(m="test", allow_empty=True) + repo.git.submodule("add", "../module", "module") + repo.index.commit("add submodule") + + cloned_repo_path = osp.join(rwdir, "cloned_repo") + cloned_repo = git.Repo.clone_from(repo_path, cloned_repo_path) + + cloned_repo.submodule_update(init=True, recursive=True) + + @with_rw_directory + @_patch_git_config("protocol.file.allow", "always") + def test_update_rejects_parent_component_in_name(self, rwdir): + source = git.Repo.init(osp.join(rwdir, "source")) + source.git.commit(m="initial commit", allow_empty=True) + + parent = git.Repo.init(osp.join(rwdir, "parent")) + parent.git.submodule("add", source.working_tree_dir, "module") + parent.index.commit("add submodule") + modules_file = Path(parent.working_tree_dir) / ".gitmodules" + modules_file.write_text( + modules_file.read_text().replace('submodule "module"', 'submodule "../../../escaped/module"') + ) + parent.index.add([".gitmodules"]) + parent.index.commit("change submodule name") + + clone = git.Repo.clone_from(parent.working_tree_dir, osp.join(rwdir, "clone")) + with pytest.raises(ValueError, match="submodule name"): + clone.submodules[0].update(init=True) + assert not osp.exists(osp.join(rwdir, "escaped")) + + Path(rwdir, "escaped").mkdir() + git.Repo.clone_from( + source.working_tree_dir, + osp.join(clone.working_tree_dir, "module"), + separate_git_dir=osp.join(rwdir, "escaped", "module"), + allow_unsafe_options=True, + ) + with pytest.raises(ValueError, match="submodule name"): + clone.submodules[0].update(init=True) + + invalid_names = ( + "", + "..", + "../module", + R"..\module", + "nested/../module", + R"nested\..\module", + "/module", + R"\module", + "C:module", + R"C:\module", + ) + for name in invalid_names: + with pytest.raises(ValueError, match="submodule name"): + Submodule._module_abspath(clone, "module", name) + + with mock.patch.object(Submodule, "_need_gitfile_submodules", return_value=False): + with pytest.raises(ValueError, match="submodule name"): + Submodule._module_abspath(clone, "module", "../module") + + @with_rw_directory + @_patch_git_config("protocol.file.allow", "always") + def test_root_update_keeps_going_after_invalid_submodule_name(self, rwdir): + source = git.Repo.init(osp.join(rwdir, "source")) + source.git.commit(m="initial commit", allow_empty=True) + + parent = git.Repo.init(osp.join(rwdir, "parent")) + parent.git.submodule("add", source.working_tree_dir, "invalid") + parent.git.submodule("add", source.working_tree_dir, "valid") + modules_file = Path(parent.working_tree_dir) / ".gitmodules" + modules_file.write_text(modules_file.read_text().replace('submodule "invalid"', 'submodule "../invalid"')) + parent.index.add([".gitmodules"]) + parent.index.commit("add submodules") + + clone = git.Repo.clone_from(parent.working_tree_dir, osp.join(rwdir, "clone")) + assert [sm.name for sm in clone.submodules] == ["../invalid", "valid"] + + clone.submodule_update(keep_going=True) + + assert os.listdir(osp.join(clone.working_tree_dir, "invalid")) == [] + assert not clone.submodule("../invalid").module_exists() + assert clone.submodule("valid").module_exists() + + clone.submodule("../invalid").update(recursive=True, keep_going=True) + @with_rw_directory @_patch_git_config("protocol.file.allow", "always") def test_list_only_valid_submodules(self, rwdir): @@ -808,8 +1072,8 @@ def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): smm.git.add(Git.polish_url(fp)) smm.git.commit(m="new file added") - # Submodules are retrieved from the current commit's tree, therefore we can't really get a new submodule - # object pointing to the new submodule commit. + # Submodules are retrieved from the current commit's tree, therefore we can't + # really get a new submodule object pointing to the new submodule commit. sm_too = parent.submodules["module_moved"] assert parent.head.commit.tree[sm.path].binsha == sm.binsha assert sm_too.binsha == sm.binsha, "cached submodule should point to the same commit as updated one" @@ -848,8 +1112,9 @@ def assert_exists(sm, value=True): # END assert_exists - # As git is backwards compatible itself, it would still recognize what we do here... unless we really - # muss it up. That's the only reason why the test is still here... + # As git is backwards compatible itself, it would still recognize what we do + # here... unless we really muss it up. That's the only reason why the test is + # still here... assert len(parent.git.submodule().splitlines()) == 1 module_repo_path = osp.join(sm.module().working_tree_dir, ".git") @@ -885,7 +1150,8 @@ def assert_exists(sm, value=True): assert_exists(csm) # Rename nested submodule. - # This name would move itself one level deeper - needs special handling internally. + # This name would move itself one level deeper - needs special handling + # internally. new_name = csm.name + "/mine" assert csm.rename(new_name).name == new_name assert_exists(csm) @@ -901,7 +1167,7 @@ def assert_exists(sm, value=True): csm.repo.index.commit("Have to commit submodule change for algorithm to pick it up") assert csm.url == "bar" - self.assertRaises( + self.assertRaises( # noqa: B017 Exception, rsm.update, recursive=True, @@ -942,6 +1208,22 @@ def test_ignore_non_submodule_file(self, rwdir): assert len(parent.submodules) == 0 + @with_rw_directory + def test_gitmodules_does_not_merge_includes(self, rwdir): + parent = git.Repo.init(rwdir) + secret_path = osp.join(rwdir, "secret") + with open(secret_path, "w", encoding="utf-8") as secret: + secret.write("not git config\n") + with open(osp.join(rwdir, ".gitmodules"), "w", encoding="utf-8") as modules: + modules.write('[submodule "module"]\n') + modules.write("\tpath = module\n") + modules.write("\turl = https://example.com/module.git\n") + modules.write("[include]\n") + modules.write("\tpath = %s\n" % secret_path) + + parser = Submodule._config_parser(parent, None, read_only=True) + self.assertEqual(parser.get_value('submodule "module"', "path"), "module") + @with_rw_directory def test_remove_norefs(self, rwdir): parent = git.Repo.init(osp.join(rwdir, "parent")) @@ -973,8 +1255,9 @@ def test_rename(self, rwdir): # This is needed to work around a PermissionError on Windows, resembling others, # except new in Python 3.12. (*Maybe* this could be due to changes in CPython's # garbage collector detailed in https://github.com/python/cpython/issues/97922.) - if os.name == "nt" and sys.version_info >= (3, 12): + if sys.platform == "win32" and sys.version_info >= (3, 12): gc.collect() + gc.collect() # Some finalizer scenarios need two collections, at least in theory. new_path = "renamed/myname" assert sm.move(new_path).name == new_path @@ -1011,13 +1294,15 @@ def test_branch_renames(self, rw_dir): sm_source_repo.index.add([new_file]) sm.repo.index.commit("added new file") - # Change designated submodule checkout branch to the new upstream feature branch. + # Change designated submodule checkout branch to the new upstream feature + # branch. with sm.config_writer() as smcw: smcw.set_value("branch", sm_fb.name) assert sm.repo.is_dirty(index=True, working_tree=False) sm.repo.index.commit("changed submodule branch to '%s'" % sm_fb) - # Verify submodule update with feature branch that leaves currently checked out branch in it's past. + # Verify submodule update with feature branch that leaves currently checked out + # branch in it's past. sm_mod = sm.module() prev_commit = sm_mod.commit() assert sm_mod.head.ref.name == "master" @@ -1029,7 +1314,8 @@ def test_branch_renames(self, rw_dir): assert sm_mod.head.ref.name == sm_fb.name assert sm_mod.commit() == sm_fb.commit - # Create new branch which is in our past, and thus seemingly unrelated to the currently checked out one. + # Create new branch which is in our past, and thus seemingly unrelated to the + # currently checked out one. # To make it even 'harder', we shall fork and create a new commit. sm_pfb = sm_source_repo.create_head("past-feature", commit="HEAD~20") sm_pfb.checkout() @@ -1043,8 +1329,8 @@ def test_branch_renames(self, rw_dir): # Test submodule updates - must fail if submodule is dirty. touch(osp.join(sm_mod.working_tree_dir, "unstaged file")) - # This doesn't fail as our own submodule binsha didn't change, and the reset is only triggered if - # to_latest_revision is True. + # This doesn't fail as our own submodule binsha didn't change, and the reset is + # only triggered if to_latest_revision is True. parent_repo.submodule_update(to_latest_revision=False) assert sm_mod.head.ref.name == sm_pfb.name, "should have been switched to past head" assert sm_mod.commit() == sm_fb.commit, "Head wasn't reset" @@ -1054,7 +1340,7 @@ def test_branch_renames(self, rw_dir): assert sm_mod.commit() == sm_pfb.commit, "Now head should have been reset" assert sm_mod.head.ref.name == sm_pfb.name - @skipUnless(os.name == "nt", "Specifically for Windows.") + @skipUnless(sys.platform == "win32", "Specifically for Windows.") def test_to_relative_path_with_super_at_root_drive(self): class Repo: working_tree_dir = "D:\\" @@ -1065,6 +1351,61 @@ class Repo: msg = '_to_relative_path should be "submodule_path" but was "%s"' % relative_path assert relative_path == "submodule_path", msg + @with_rw_directory + def test_to_relative_path_is_confined_to_parent(self, rwdir): + class Repo: + working_tree_dir = osp.join(rwdir, "parent") + + os.makedirs(osp.join(Repo.working_tree_dir, "nested")) + + assert Submodule._to_relative_path(Repo(), osp.join(Repo.working_tree_dir, "nested")) == "nested" + assert Submodule._to_relative_path(Repo(), osp.join("nested", "..", "module")) == "module" + self.assertRaises(ValueError, Submodule._to_relative_path, Repo(), Repo.working_tree_dir) + self.assertRaises(ValueError, Submodule._to_relative_path, Repo(), osp.join("..", "outside")) + self.assertRaises( + ValueError, + Submodule._to_relative_path, + Repo(), + osp.join(Repo.working_tree_dir + "-other", "module"), + ) + + @skipUnless(sys.platform == "win32", "Specifically for Windows.") + @with_rw_directory + def test_to_relative_path_windows_path_kinds(self, rwdir): + class Repo: + working_tree_dir = osp.join(rwdir, "parent") + + inside_path = osp.join(Repo.working_tree_dir, "nested", "module") + _drive, rooted_inside_path = osp.splitdrive(inside_path) + + assert Submodule._to_relative_path(Repo(), rooted_inside_path) == "nested/module" + assert Submodule._to_relative_path(Repo(), rooted_inside_path.replace("\\", "/")) == "nested/module" + assert Submodule._to_relative_path(Repo(), PathLikeMock(inside_path)) == "nested/module" + self.assertRaises( + ValueError, + Submodule._to_relative_path, + Repo(), + f"{osp.splitdrive(Repo.working_tree_dir)[0]}relative", + ) + self.assertRaises(ValueError, Submodule._to_relative_path, Repo(), R"Z:\outside") + self.assertRaises(ValueError, Submodule._to_relative_path, Repo(), R"\\server\share\outside") + self.assertRaises(ValueError, Submodule._to_relative_path, Repo(), R"\\?\C:\outside") + + @skipUnless(sys.platform == "win32", "Specifically for Windows.") + @with_rw_directory + def test_find_submodule_git_dir_with_drive_rooted_gitfile(self, rwdir): + with cwd(rwdir): + module = git.Repo.init(osp.join(rwdir, "module")) + _drive, rooted_git_dir = osp.splitdrive(module.git_dir) + gitfile = osp.join(rwdir, "gitfile") + with open(gitfile, "w") as stream: + stream.write(f"gitdir: {rooted_git_dir}") + + resolved_git_dir = find_submodule_git_dir(gitfile) + + assert resolved_git_dir is not None + assert osp.samefile(resolved_git_dir, module.git_dir) + @pytest.mark.xfail( reason="for some unknown reason the assertion fails, even though it in fact is working in more common setup", raises=AssertionError, @@ -1160,6 +1501,20 @@ def test_add_no_clone_multi_options_argument(self, rwdir): with self.assertRaises(cp.NoOptionError): sm_config.get_value("core", "eol") + @with_rw_directory + def test_add_does_not_expand_environment_variables_in_url(self, rwdir): + parent = git.Repo.init(osp.join(rwdir, "parent")) + url = osp.join(rwdir, "${GITPYTHON_TEST_SECRET}") + source = git.Repo.init(url) + Path(source.working_tree_dir, "file").write_text("content") + source.index.add(["file"]) + source.index.commit("initial") + + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + sm = Submodule.add(parent, "new", "new", url) + + assert sm.url == Git.polish_url(url, expand_vars=False) + @with_rw_repo("HEAD") def test_submodule_add_unsafe_url(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: @@ -1184,8 +1539,8 @@ def test_submodule_add_unsafe_url_allowed(self, rw_repo): "fd::/foo", ] for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): Submodule.add(rw_repo, "new", "new", url, allow_unsafe_protocols=True) assert not tmp_file.exists() @@ -1269,8 +1624,8 @@ def test_submodule_update_unsafe_url_allowed(self, rw_repo): ] for url in urls: submodule = Submodule(rw_repo, b"\0" * 20, name="new", path="new", url=url) - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): submodule.update(allow_unsafe_protocols=True) assert not tmp_file.exists() @@ -1292,6 +1647,17 @@ def test_submodule_update_unsafe_options(self, rw_repo): submodule.update(clone_multi_options=[unsafe_option]) assert not tmp_file.exists() + @with_rw_repo("HEAD") + def test_submodule_update_unsafe_options_are_checked_after_splitting_multi_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = Path(tdir) + payload = "--single-branch --config protocol.ext.allow=always" + submodule = Submodule(rw_repo, b"\0" * 20, name="new", path="new", url=str(tmp_dir)) + + with self.assertRaises(UnsafeOptionError): + submodule.update(clone_multi_options=[payload]) + assert not submodule.module_exists() + @with_rw_repo("HEAD") def test_submodule_update_unsafe_options_allowed(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: @@ -1316,3 +1682,23 @@ def test_submodule_update_unsafe_options_allowed(self, rw_repo): for unsafe_option in unsafe_options: with self.assertRaises(GitCommandError): submodule.update(clone_multi_options=[unsafe_option], allow_unsafe_options=True) + + @with_rw_directory + @_patch_git_config("protocol.file.allow", "always") + def test_submodule_update_relative_url(self, rwdir): + parent_path = osp.join(rwdir, "parent") + parent_repo = git.Repo.init(parent_path) + submodule_path = osp.join(rwdir, "module") + submodule_repo = git.Repo.init(submodule_path) + submodule_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo.git.submodule("add", "../module", "module") + parent_repo.index.commit("add submodule with relative URL") + + cloned_path = osp.join(rwdir, "cloned_repo") + cloned_repo = git.Repo.clone_from(parent_path, cloned_path) + + cloned_repo.submodule_update(init=True, recursive=True) + + has_module = any(sm.name == "module" for sm in cloned_repo.submodules) + assert has_module, "Relative submodule was not updated properly" diff --git a/test/test_tree.py b/test/test_tree.py index 7713413a6..629fd4d32 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -4,11 +4,18 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO +import os.path as osp +from pathlib import Path +import subprocess -from git.objects import Tree, Blob -from test.lib import TestBase +import pytest -import os.path as osp +from git.objects import Blob, Tree +from git.repo import Repo +from git.util import cwd + +from test.lib import TestBase, with_rw_directory +from .lib.helper import PathLikeMock, with_rw_repo class TestTree(TestBase): @@ -40,6 +47,70 @@ def test_serializable(self): testtree._deserialize(stream) # END for each item in tree + @with_rw_directory + def _get_git_ordered_files(self, rw_dir): + """Get files as git orders them, to compare in test_tree_modifier_ordering.""" + # Create directory contents. + Path(rw_dir, "file").mkdir() + for filename in ( + "bin", + "bin.d", + "file.to", + "file.toml", + "file.toml.bin", + "file0", + ): + Path(rw_dir, filename).touch() + Path(rw_dir, "file", "a").touch() + + with cwd(rw_dir): + # Prepare the repository. + subprocess.run(["git", "init", "-q"], check=True) + subprocess.run(["git", "add", "."], check=True) + subprocess.run(["git", "commit", "-m", "c1"], check=True) + + # Get git output from which an ordered file list can be parsed. + rev_parse_command = ["git", "rev-parse", "HEAD^{tree}"] + tree_hash = subprocess.check_output(rev_parse_command).decode().strip() + cat_file_command = ["git", "cat-file", "-p", tree_hash] + cat_file_output = subprocess.check_output(cat_file_command).decode() + + return [line.split()[-1] for line in cat_file_output.split("\n") if line] + + def test_tree_modifier_ordering(self): + """TreeModifier.set_done() sorts files in the same order git does.""" + git_file_names_in_order = self._get_git_ordered_files() + + hexsha = "6c1faef799095f3990e9970bc2cb10aa0221cf9c" + roottree = self.rorepo.tree(hexsha) + blob_mode = Tree.blob_id << 12 + tree_mode = Tree.tree_id << 12 + + files_in_desired_order = [ + (blob_mode, "bin"), + (blob_mode, "bin.d"), + (blob_mode, "file.to"), + (blob_mode, "file.toml"), + (blob_mode, "file.toml.bin"), + (blob_mode, "file0"), + (tree_mode, "file"), + ] + mod = roottree.cache + for file_mode, file_name in files_in_desired_order: + mod.add(hexsha, file_mode, file_name) + # end for each file + + def file_names_in_order(): + return [t[1] for t in files_in_desired_order] + + def names_in_mod_cache(): + a = [t[2] for t in mod._cache] + here = file_names_in_order() + return [e for e in a if e in here] + + mod.set_done() + assert names_in_mod_cache() == git_file_names_in_order, "set_done() performs git-sorting" + def test_traverse(self): root = self.rorepo.tree("0.1.6") num_recursive = 0 @@ -59,12 +130,18 @@ def test_traverse(self): assert len(list(root)) == len(list(root.traverse(depth=1))) # Only choose trees. - trees_only = lambda i, d: i.type == "tree" + + def trees_only(i, _d): + return i.type == "tree" + trees = list(root.traverse(predicate=trees_only)) assert len(trees) == len([i for i in root.traverse() if trees_only(i, 0)]) # Test prune. - lib_folder = lambda t, d: t.path == "lib" + + def lib_folder(t, _d): + return t.path == "lib" + pruned_trees = list(root.traverse(predicate=trees_only, prune=lib_folder)) assert len(pruned_trees) < len(trees) @@ -88,3 +165,57 @@ def test_traverse(self): assert root[item.path] == item == root / item.path # END for each item assert found_slash + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / ".gitignore" + assert isinstance(blob, Blob) + assert blob.hexsha == "787b3d442a113b78e343deb585ab5531eb7187fa" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / PathLikeMock(".gitignore") + assert isinstance(blob, Blob) + assert blob.hexsha == "787b3d442a113b78e343deb585ab5531eb7187fa" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_invalid_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + with pytest.raises(KeyError): + repo.tree() / "doesnotexist" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_invalid_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + with pytest.raises(KeyError): + repo.tree() / PathLikeMock("doesnotexist") + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_nested_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / "git/__init__.py" + assert isinstance(blob, Blob) + assert blob.hexsha == "d87dcbdbb65d2782e14eea27e7f833a209c052f3" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_nested_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / PathLikeMock("git/__init__.py") + assert isinstance(blob, Blob) + assert blob.hexsha == "d87dcbdbb65d2782e14eea27e7f833a209c052f3" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_folder_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + tree = repo.tree() / "git" + assert isinstance(tree, Tree) + assert tree.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_folder_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + tree = repo.tree() / PathLikeMock("git") + assert isinstance(tree, Tree) + assert tree.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" diff --git a/test/test_util.py b/test/test_util.py index f3769088c..cf4299d5a 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -34,11 +34,13 @@ LockFile, cygpath, decygpath, + is_cygwin_git, get_user_id, remove_password_if_present, rmtree, ) -from test.lib import TestBase, with_rw_repo + +from test.lib import TestBase, requires_symlinks, with_rw_repo @pytest.fixture @@ -51,7 +53,9 @@ def permission_error_tmpdir(tmp_path): # Set up PermissionError on Windows, where we can't delete read-only files. (td / "x").chmod(stat.S_IRUSR) - # Set up PermissionError on Unix, where we can't delete files in read-only directories. + # Set up PermissionError on Unix, where non-root users can't delete files in + # read-only directories. (Tests that rely on this and assert that rmtree raises + # PermissionError will fail if they are run as root.) td.chmod(stat.S_IRUSR | stat.S_IXUSR) yield td @@ -109,7 +113,8 @@ def test_deletes_dir_with_readonly_files(self, tmp_path): sys.platform == "cygwin", reason="Cygwin can't set the permissions that make the test meaningful.", ) - def test_avoids_changing_permissions_outside_tree(self, tmp_path): + @requires_symlinks + def test_avoids_changing_permissions_outside_tree(self, tmp_path, request): # Automatically works on Windows, but on Unix requires either special handling # or refraining from attempting to fix PermissionError by making chmod calls. @@ -121,9 +126,32 @@ def test_avoids_changing_permissions_outside_tree(self, tmp_path): dir2 = tmp_path / "dir2" dir2.mkdir() - (dir2 / "symlink").symlink_to(dir1 / "file") + symlink = dir2 / "symlink" + symlink.symlink_to(dir1 / "file") dir2.chmod(stat.S_IRUSR | stat.S_IXUSR) + def preen_dir2(): + """Don't leave unwritable directories behind. + + pytest has difficulties cleaning up after the fact on some platforms, + e.g., macOS, and whines incessantly until the issue is resolved--regardless + of the pytest session. + """ + rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + if not dir2.exists(): + return + if symlink.exists(): + try: + # Try lchmod first, if the platform supports it. + symlink.lchmod(rwx) + except NotImplementedError: + # The platform (probably win32) doesn't support lchmod; fall back to chmod. + symlink.chmod(rwx) + dir2.chmod(rwx) + rmtree(dir2) + + request.addfinalizer(preen_dir2) + try: rmtree(dir2) except PermissionError: @@ -147,11 +175,12 @@ def _patch_for_wrapping_test(self, mocker, hide_windows_known_errors): mocker.patch.object(pathlib.Path, "chmod") @pytest.mark.skipif( - os.name != "nt", + sys.platform != "win32", reason="PermissionError is only ever wrapped on Windows", ) def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): - """rmtree wraps PermissionError on Windows when HIDE_WINDOWS_KNOWN_ERRORS is true.""" + """rmtree wraps PermissionError on Windows when HIDE_WINDOWS_KNOWN_ERRORS is + true.""" self._patch_for_wrapping_test(mocker, True) with pytest.raises(SkipTest): @@ -165,11 +194,12 @@ def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): "hide_windows_known_errors", [ pytest.param(False), - pytest.param(True, marks=pytest.mark.skipif(os.name == "nt", reason="We would wrap on Windows")), + pytest.param(True, marks=pytest.mark.skipif(sys.platform == "win32", reason="We would wrap on Windows")), ], ) def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir, hide_windows_known_errors): - """rmtree does not wrap PermissionError on non-Windows systems or when HIDE_WINDOWS_KNOWN_ERRORS is false.""" + """rmtree does not wrap PermissionError on non-Windows systems or when + HIDE_WINDOWS_KNOWN_ERRORS is false.""" self._patch_for_wrapping_test(mocker, hide_windows_known_errors) with pytest.raises(PermissionError): @@ -180,7 +210,9 @@ def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_ @pytest.mark.parametrize("hide_windows_known_errors", [False, True]) def test_does_not_wrap_other_errors(self, tmp_path, mocker, hide_windows_known_errors): - file_not_found_tmpdir = tmp_path / "testdir" # It is deliberately never created. + # The file is deliberately never created. + file_not_found_tmpdir = tmp_path / "testdir" + self._patch_for_wrapping_test(mocker, hide_windows_known_errors) with pytest.raises(FileNotFoundError): @@ -208,7 +240,7 @@ def _run_parse(name, value): return ast.literal_eval(output) @pytest.mark.skipif( - os.name != "nt", + sys.platform != "win32", reason="These environment variables are only used on Windows.", ) @pytest.mark.parametrize( @@ -342,6 +374,24 @@ def test_decygpath(self, wpath, cpath): assert wcpath == wpath.replace("/", "\\"), cpath +class TestIsCygwinGit: + """Tests for :func:`is_cygwin_git`""" + + def test_on_path_executable(self): + # Currently we assume tests run on Cygwin use Cygwin git. See #533 and #1455 for background. + if sys.platform == "cygwin": + assert is_cygwin_git("git") + else: + assert not is_cygwin_git("git") + + def test_none_executable(self): + assert not is_cygwin_git(None) + + def test_with_missing_uname(self): + """Test for handling when `uname` isn't in the same directory as `git`""" + assert not is_cygwin_git("/bogus_path/git") + + class _Member: """A member of an IterableList.""" @@ -358,6 +408,10 @@ def __repr__(self): class TestUtils(TestBase): """Tests for most utilities in :mod:`git.util`.""" + @pytest.mark.skipif(os.name != "nt", reason="Specifically for Windows drive-rooted paths.") + def test_cygpath_drive_rooted_path(self): + assert cygpath(R"\directory\file") == "/directory/file" + def test_it_should_dashify(self): self.assertEqual("this-is-my-argument", dashify("this_is_my_argument")) self.assertEqual("foo", dashify("foo")) @@ -404,7 +458,7 @@ def test_blocking_lock_file(self): elapsed = time.time() - start extra_time = 0.02 - if os.name == "nt" or sys.platform == "cygwin": + if sys.platform in {"win32", "cygwin"}: extra_time *= 6 # Without this, we get indeterministic failures on Windows. elif sys.platform == "darwin": extra_time *= 18 # The situation on macOS is similar, but with more delay. @@ -500,7 +554,8 @@ def test_actor_get_uid_laziness_called(self, mock_get_uid): committer = Actor.committer(None) author = Actor.author(None) # We can't test with `self.rorepo.config_reader()` here, as the UUID laziness - # depends on whether the user running the test has their global user.name config set. + # depends on whether the user running the test has their global user.name config + # set. self.assertEqual(committer.name, "user") self.assertTrue(committer.email.startswith("user@")) self.assertEqual(author.name, "user") @@ -621,6 +676,7 @@ def test_pickle_tzoffset(self): def test_remove_password_from_command_line(self): username = "fakeuser" password = "fakepassword1234" + authorization = "Bearer fake-token-1234" url_with_user_and_pass = "https://{}:{}@fakerepo.example.com/testrepo".format(username, password) url_with_user = "https://{}@fakerepo.example.com/testrepo".format(username) url_with_pass = "https://:{}@fakerepo.example.com/testrepo".format(password) @@ -631,6 +687,7 @@ def test_remove_password_from_command_line(self): cmd_3 = ["git", "clone", "-v", url_with_pass] cmd_4 = ["git", "clone", "-v", url_without_user_or_pass] cmd_5 = ["no", "url", "in", "this", "one"] + cmd_6 = ["git", "-c", "http.extraHeader=Authorization: %s" % authorization, "fetch"] redacted_cmd_1 = remove_password_if_present(cmd_1) assert username not in " ".join(redacted_cmd_1) @@ -650,3 +707,7 @@ def test_remove_password_from_command_line(self): assert cmd_4 == remove_password_if_present(cmd_4) assert cmd_5 == remove_password_if_present(cmd_5) + + redacted_cmd_6 = remove_password_if_present(cmd_6) + assert authorization not in " ".join(redacted_cmd_6) + assert "http.extraHeader=Authorization: *****" in redacted_cmd_6 diff --git a/tox.ini b/tox.ini index f9ac25b78..fc3f821d6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -env_list = py{37,38,39,310,311,312}, lint, mypy, html +env_list = py{37,38,39,310,311,312,313,314,315}, ruff, format, mypy, html, misc [testenv] description = Run unit tests @@ -9,24 +9,42 @@ extras = test pass_env = SSH_* commands = pytest --color=yes {posargs} -[testenv:lint] -description = Lint via pre-commit +[testenv:ruff] +description = Lint with Ruff base_python = py{39,310,311,312,38,37} +deps = ruff set_env = - SKIP = black-format -commands = pre-commit run --all-files --hook-stage manual + CLICOLOR_FORCE = 1 # Set NO_COLOR to override this. +commands = ruff check . + +[testenv:format] +description = Check formatting with Ruff +base_python = py{39,310,311,312,38,37} +deps = ruff +set_env = + CLICOLOR_FORCE = 1 # Set NO_COLOR to override this. +commands = ruff format --check . [testenv:mypy] description = Typecheck with mypy base_python = py{39,310,311,312,38,37} -commands = mypy -p git +set_env = + MYPY_FORCE_COLOR = 1 +commands = mypy ignore_outcome = true [testenv:html] description = Build HTML documentation base_python = py{39,310,311,312,38,37} -deps = -r doc/requirements.txt +extras = doc allowlist_externals = make commands = make BUILDDIR={env_tmp_dir}/doc/build -C doc clean make BUILDDIR={env_tmp_dir}/doc/build -C doc html + +[testenv:misc] +description = Run other checks via pre-commit +base_python = py{39,310,311,312,38,37} +set_env = + SKIP = ruff-format,ruff +commands = pre-commit run --all-files