From 4c86955a41f8d49d6c3719b61a0607aae7c866f8 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 27 Jun 2025 13:06:09 -0400 Subject: [PATCH 01/27] Add support for JIT binaries --- backend/scripts/populate_binaries.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/scripts/populate_binaries.py b/backend/scripts/populate_binaries.py index 4011053..facb239 100644 --- a/backend/scripts/populate_binaries.py +++ b/backend/scripts/populate_binaries.py @@ -110,6 +110,15 @@ def get_standard_binaries(): "icon": "search", "display_order": 7, }, + { + "id": "jit", + "name": "JIT Build", + "flags": ["--enable-experimental-jit=yes"], + "description": "Just-In-Time compilation enabled.", + "color": "#fc03df", + "icon": "zap", + "display_order": 8, + }, ] From 121e1f6832cfd14f9d659fdeee90975b5e04d7c0 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 27 Jun 2025 13:40:58 -0400 Subject: [PATCH 02/27] Update backend/scripts/populate_binaries.py Co-authored-by: Brandt Bucher --- backend/scripts/populate_binaries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/scripts/populate_binaries.py b/backend/scripts/populate_binaries.py index facb239..6ed7bb5 100644 --- a/backend/scripts/populate_binaries.py +++ b/backend/scripts/populate_binaries.py @@ -113,7 +113,7 @@ def get_standard_binaries(): { "id": "jit", "name": "JIT Build", - "flags": ["--enable-experimental-jit=yes"], + "flags": ["--enable-experimental-jit"], "description": "Just-In-Time compilation enabled.", "color": "#fc03df", "icon": "zap", From 598302b7b4dfaecc2f0945b8c2d335128c3da6fa Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 27 Jun 2025 13:46:45 -0400 Subject: [PATCH 03/27] Add LLVM to the environment --- .github/workflows/benchmark.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f412573..f4a44a4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -31,6 +31,10 @@ on: description: 'Make flags for CPython build' required: false default: '-j' + llvm: + description: 'LLVM version to use' + required: false + default: '19' jobs: benchmark: @@ -61,8 +65,11 @@ jobs: # Install CPython dependencies using their script cd cpython sudo .github/workflows/posix-deps-apt.sh - - # Install Memray dependencies + + # Install JIT dependencies + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ inputs.llvm }} + + # Install Memray dependencies sudo apt-get install -y \ python3-dev \ libdebuginfod-dev \ @@ -73,6 +80,8 @@ jobs: env: MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }} run: | + export PATH="$(llvm-config-${{ inputs.llvm }} --bindir):$PATH" + # Build command with conditional flags CMD="memory-tracker benchmark '${{ github.event.inputs.commit_range }}'" CMD="$CMD --repo-path ./cpython" From 232158f071a224924c08e676529f302338681340 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 27 Jun 2025 14:06:46 -0400 Subject: [PATCH 04/27] Only install LLVM for JIT builds --- .github/workflows/benchmark.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f4a44a4..0664116 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -67,7 +67,9 @@ jobs: sudo .github/workflows/posix-deps-apt.sh # Install JIT dependencies - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ inputs.llvm }} + if [ "${{ inputs.binary_id }}" = "jit" ]; then + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ inputs.llvm }} + fi # Install Memray dependencies sudo apt-get install -y \ @@ -80,7 +82,9 @@ jobs: env: MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }} run: | - export PATH="$(llvm-config-${{ inputs.llvm }} --bindir):$PATH" + if [ "${{ inputs.binary_id }}" = "jit" ]; then + export PATH="$(llvm-config-${{ inputs.llvm }} --bindir):$PATH" + fi # Build command with conditional flags CMD="memory-tracker benchmark '${{ github.event.inputs.commit_range }}'" From b55fd295237874552d685bd8bb22af2a126aea14 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 28 Jun 2025 19:00:29 +0100 Subject: [PATCH 05/27] Daily runs --- .github/workflows/daily-benchmark.yml | 174 ++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 .github/workflows/daily-benchmark.yml diff --git a/.github/workflows/daily-benchmark.yml b/.github/workflows/daily-benchmark.yml new file mode 100644 index 0000000..b6a5592 --- /dev/null +++ b/.github/workflows/daily-benchmark.yml @@ -0,0 +1,174 @@ +name: Daily Memory Tracker Benchmark + +on: + schedule: + # Run daily at 02:00 UTC + - cron: '0 2 * * *' + workflow_dispatch: + inputs: + target_date: + description: 'Date to get commits from (YYYY-MM-DD, defaults to today)' + required: false + type: string + binary_id: + description: 'Binary ID to use for benchmarking' + required: false + default: 'default' + environment_id: + description: 'Environment ID' + required: false + default: 'gcc-11' + server_url: + description: 'Memory tracker server URL' + required: false + default: 'https://memory.python.org' + cpython_repo: + description: 'CPython repository URL' + required: false + default: 'https://github.com/python/cpython.git' + +jobs: + get-daily-commits: + runs-on: ubuntu-latest + outputs: + commits: ${{ steps.get-commits.outputs.commits }} + commit-count: ${{ steps.get-commits.outputs.commit-count }} + + steps: + - name: Clone CPython repository + run: | + git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython + cd cpython + git fetch --all + + - name: Get commits from target date + id: get-commits + run: | + cd cpython + + # Determine target date + if [ -n "${{ github.event.inputs.target_date }}" ]; then + TARGET_DATE="${{ github.event.inputs.target_date }}" + else + TARGET_DATE=$(date -u +%Y-%m-%d) + fi + + echo "Getting commits from date: $TARGET_DATE" + + # Get commits from the target date (00:00 to 23:59 UTC) + COMMITS=$(git log --since="$TARGET_DATE 00:00:00 UTC" --until="$TARGET_DATE 23:59:59 UTC" --pretty=format:"%H" --reverse) + + if [ -z "$COMMITS" ]; then + echo "No commits found for date $TARGET_DATE" + echo "commits=" >> $GITHUB_OUTPUT + echo "commit-count=0" >> $GITHUB_OUTPUT + else + # Convert to JSON array format for matrix strategy + COMMITS_JSON=$(echo "$COMMITS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + COMMIT_COUNT=$(echo "$COMMITS" | wc -l) + + echo "Found $COMMIT_COUNT commits for date $TARGET_DATE" + echo "commits=$COMMITS_JSON" >> $GITHUB_OUTPUT + echo "commit-count=$COMMIT_COUNT" >> $GITHUB_OUTPUT + + echo "Commits to benchmark:" + echo "$COMMITS" + fi + + benchmark-commits: + needs: get-daily-commits + if: needs.get-daily-commits.outputs.commit-count > 0 + runs-on: ubuntu-latest + strategy: + matrix: + commit: ${{ fromJson(needs.get-daily-commits.outputs.commits) }} + fail-fast: false + max-parallel: 3 + + steps: + - name: Checkout memory tracker + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Clone CPython repository + run: | + git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython + cd cpython + git fetch --depth=200 + + - name: Install memory tracker worker + run: | + cd worker + pip install -e . + + - name: Install build dependencies + run: | + # Install CPython dependencies using their script + cd cpython + sudo .github/workflows/posix-deps-apt.sh + + # Install Memray dependencies + sudo apt-get install -y \ + python3-dev \ + libdebuginfod-dev \ + libunwind-dev \ + liblz4-dev + + - name: Run memory benchmark for commit + env: + MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }} + run: | + COMMIT="${{ matrix.commit }}" + + # Build command for single commit + CMD="memory-tracker benchmark '$COMMIT'" + CMD="$CMD --repo-path ./cpython" + CMD="$CMD --binary-id '${{ github.event.inputs.binary_id || 'default' }}'" + CMD="$CMD --environment-id '${{ github.event.inputs.environment_id || 'gcc-11' }}'" + CMD="$CMD --api-base '${{ github.event.inputs.server_url || 'https://memory.python.org' }}'" + CMD="$CMD --output-dir ./benchmark_results" + CMD="$CMD --force" + CMD="$CMD -vv" + + echo "Running benchmark for commit: $COMMIT" + echo "Command: $CMD" + eval $CMD + + - name: Upload benchmark results (if failed) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: benchmark-logs-${{ matrix.commit }} + path: | + *.log + ./benchmark_results/ + retention-days: 7 + + - name: Upload benchmark results (on success) + if: success() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ matrix.commit }} + path: ./benchmark_results/ + retention-days: 30 + + summary: + needs: [get-daily-commits, benchmark-commits] + if: always() + runs-on: ubuntu-latest + + steps: + - name: Print summary + run: | + echo "Daily benchmark run completed" + echo "Total commits processed: ${{ needs.get-daily-commits.outputs.commit-count }}" + + if [ "${{ needs.get-daily-commits.outputs.commit-count }}" = "0" ]; then + echo "No commits found for the target date" + else + echo "Benchmark jobs completed with status: ${{ needs.benchmark-commits.result }}" + fi \ No newline at end of file From b751ecadc0c96a5c11580ebdc2ec74386506d8d3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 28 Jun 2025 19:08:13 +0100 Subject: [PATCH 06/27] Add JIT to daily runs and reschedule the upload --- .claude/settings.local.json | 22 ---- .github/workflows/daily-benchmark.yml | 153 ++++++++++++++------------ 2 files changed, 82 insertions(+), 93 deletions(-) delete mode 100644 .claude/settings.local.json diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index ccf5026..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(rm:*)", - "Bash(grep:*)", - "Bash(npm run build:*)", - "Bash(curl:*)", - "Bash(npm run lint:*)", - "Bash(sed:*)", - "Bash(find:*)", - "Bash(npm run typecheck:*)", - "Bash(npm install)", - "Bash(ls:*)", - "Bash(npx next lint:*)", - "Bash(npx tsc:*)", - "Bash(rg:*)", - "Bash(npm run type-check:*)", - "Bash(npx next build:*)" - ], - "deny": [] - } -} \ No newline at end of file diff --git a/.github/workflows/daily-benchmark.yml b/.github/workflows/daily-benchmark.yml index b6a5592..42488bb 100644 --- a/.github/workflows/daily-benchmark.yml +++ b/.github/workflows/daily-benchmark.yml @@ -2,22 +2,18 @@ name: Daily Memory Tracker Benchmark on: schedule: - # Run daily at 02:00 UTC - - cron: '0 2 * * *' + # Run daily at 23:00 UTC (EOD) to pick up all commits from the day + - cron: '0 23 * * *' workflow_dispatch: inputs: target_date: description: 'Date to get commits from (YYYY-MM-DD, defaults to today)' required: false type: string - binary_id: - description: 'Binary ID to use for benchmarking' - required: false - default: 'default' environment_id: description: 'Environment ID' required: false - default: 'gcc-11' + default: 'gh_actions' server_url: description: 'Memory tracker server URL' required: false @@ -26,26 +22,54 @@ on: description: 'CPython repository URL' required: false default: 'https://github.com/python/cpython.git' + llvm: + description: 'LLVM version to use for JIT builds' + required: false + default: '19' jobs: - get-daily-commits: + benchmark-builds: runs-on: ubuntu-latest - outputs: - commits: ${{ steps.get-commits.outputs.commits }} - commit-count: ${{ steps.get-commits.outputs.commit-count }} + strategy: + matrix: + build_config: + - binary_id: 'default' + configure_flags: '-C' + description: 'Default build' + install_deps: 'standard' + - binary_id: 'debug' + configure_flags: '--with-pydebug' + description: 'Debug build' + install_deps: 'standard' + - binary_id: 'jit' + configure_flags: '--enable-experimental-jit' + description: 'JIT build' + install_deps: 'jit' + - binary_id: 'lto-pgo' + configure_flags: '--with-lto --enable-optimizations' + description: 'LTO-PGO build' + install_deps: 'standard' + - binary_id: 'nogil' + configure_flags: '--disable-gil' + description: 'Free-threaded build' + install_deps: 'standard' + fail-fast: false steps: - - name: Clone CPython repository + - name: Checkout memory tracker + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Clone CPython repository and get commits run: | git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython cd cpython git fetch --all - - name: Get commits from target date - id: get-commits - run: | - cd cpython - # Determine target date if [ -n "${{ github.event.inputs.target_date }}" ]; then TARGET_DATE="${{ github.event.inputs.target_date }}" @@ -60,45 +84,30 @@ jobs: if [ -z "$COMMITS" ]; then echo "No commits found for date $TARGET_DATE" - echo "commits=" >> $GITHUB_OUTPUT - echo "commit-count=0" >> $GITHUB_OUTPUT + exit 1 else - # Convert to JSON array format for matrix strategy - COMMITS_JSON=$(echo "$COMMITS" | jq -R -s -c 'split("\n") | map(select(length > 0))') COMMIT_COUNT=$(echo "$COMMITS" | wc -l) + FIRST_COMMIT=$(echo "$COMMITS" | head -1) + LAST_COMMIT=$(echo "$COMMITS" | tail -1) echo "Found $COMMIT_COUNT commits for date $TARGET_DATE" - echo "commits=$COMMITS_JSON" >> $GITHUB_OUTPUT - echo "commit-count=$COMMIT_COUNT" >> $GITHUB_OUTPUT + echo "COMMIT_RANGE=${FIRST_COMMIT}..${LAST_COMMIT}" >> $GITHUB_ENV + echo "COMMIT_COUNT=$COMMIT_COUNT" >> $GITHUB_ENV echo "Commits to benchmark:" echo "$COMMITS" fi - - benchmark-commits: - needs: get-daily-commits - if: needs.get-daily-commits.outputs.commit-count > 0 - runs-on: ubuntu-latest - strategy: - matrix: - commit: ${{ fromJson(needs.get-daily-commits.outputs.commits) }} - fail-fast: false - max-parallel: 3 - - steps: - - name: Checkout memory tracker - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Clone CPython repository + - name: Print environment variables run: | - git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython - cd cpython - git fetch --depth=200 + echo "=== Environment Variables ===" + echo "COMMIT_RANGE: $COMMIT_RANGE" + echo "COMMIT_COUNT: $COMMIT_COUNT" + echo "Binary ID: ${{ matrix.build_config.binary_id }}" + echo "Description: ${{ matrix.build_config.description }}" + echo "Configure flags: ${{ matrix.build_config.configure_flags }}" + echo "Install deps: ${{ matrix.build_config.install_deps }}" + echo "==========================" - name: Install memory tracker worker run: | @@ -110,6 +119,11 @@ jobs: # Install CPython dependencies using their script cd cpython sudo .github/workflows/posix-deps-apt.sh + + # Install JIT dependencies if needed + if [ "${{ matrix.build_config.install_deps }}" = "jit" ]; then + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ github.event.inputs.llvm || '19' }} + fi # Install Memray dependencies sudo apt-get install -y \ @@ -118,31 +132,33 @@ jobs: libunwind-dev \ liblz4-dev - - name: Run memory benchmark for commit + - name: Run memory benchmark for commit range - ${{ matrix.build_config.description }} env: MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }} run: | - COMMIT="${{ matrix.commit }}" - - # Build command for single commit - CMD="memory-tracker benchmark '$COMMIT'" - CMD="$CMD --repo-path ./cpython" - CMD="$CMD --binary-id '${{ github.event.inputs.binary_id || 'default' }}'" - CMD="$CMD --environment-id '${{ github.event.inputs.environment_id || 'gcc-11' }}'" - CMD="$CMD --api-base '${{ github.event.inputs.server_url || 'https://memory.python.org' }}'" - CMD="$CMD --output-dir ./benchmark_results" - CMD="$CMD --force" - CMD="$CMD -vv" + if [ "${{ matrix.build_config.install_deps }}" = "jit" ]; then + export PATH="$(llvm-config-${{ github.event.inputs.llvm || '19' }} --bindir):$PATH" + export LLVM_VERSION="${{ github.event.inputs.llvm || '19' }}" + echo "LLVM Path: $(llvm-config-${{ github.event.inputs.llvm || '19' }} --bindir)" + echo "Clang version: $(clang-${{ github.event.inputs.llvm || '19' }} --version || echo 'clang-${{ github.event.inputs.llvm || '19' }} not found')" + fi - echo "Running benchmark for commit: $COMMIT" - echo "Command: $CMD" - eval $CMD + # Build command for commit range + memory-tracker benchmark "$COMMIT_RANGE" \ + --repo-path ./cpython \ + --binary-id "${{ matrix.build_config.binary_id }}" \ + --environment-id "${{ github.event.inputs.environment_id || 'gh_actions' }}" \ + --api-base "${{ github.event.inputs.server_url || 'https://memory.python.org' }}" \ + --output-dir ./benchmark_results \ + --configure-flags="${{ matrix.build_config.configure_flags }}" \ + --force \ + -vv - name: Upload benchmark results (if failed) if: failure() uses: actions/upload-artifact@v4 with: - name: benchmark-logs-${{ matrix.commit }} + name: benchmark-logs-${{ matrix.build_config.binary_id }} path: | *.log ./benchmark_results/ @@ -152,12 +168,12 @@ jobs: if: success() uses: actions/upload-artifact@v4 with: - name: benchmark-results-${{ matrix.commit }} + name: benchmark-results-${{ matrix.build_config.binary_id }} path: ./benchmark_results/ retention-days: 30 summary: - needs: [get-daily-commits, benchmark-commits] + needs: benchmark-builds if: always() runs-on: ubuntu-latest @@ -165,10 +181,5 @@ jobs: - name: Print summary run: | echo "Daily benchmark run completed" - echo "Total commits processed: ${{ needs.get-daily-commits.outputs.commit-count }}" - - if [ "${{ needs.get-daily-commits.outputs.commit-count }}" = "0" ]; then - echo "No commits found for the target date" - else - echo "Benchmark jobs completed with status: ${{ needs.benchmark-commits.result }}" - fi \ No newline at end of file + echo "Benchmark jobs completed with status: ${{ needs.benchmark-builds.result }}" + echo "Binary types benchmarked: default, debug, jit, lto-pgo, nogil" \ No newline at end of file From 5966ed46333807ff00504c2d20250afa4399578f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 28 Jun 2025 20:16:46 +0100 Subject: [PATCH 07/27] Make the server more permissive --- backend/app/routers/upload.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/backend/app/routers/upload.py b/backend/app/routers/upload.py index 1b1904f..e8a0a32 100644 --- a/backend/app/routers/upload.py +++ b/backend/app/routers/upload.py @@ -180,10 +180,23 @@ async def upload_worker_run( # Validate configure flags - the registered binary flags must be a subset of uploaded flags configure_vars = metadata.get("configure_vars", {}) uploaded_config_args = configure_vars.get("CONFIG_ARGS", "") - uploaded_flags = ( - set(uploaded_config_args.split()) if uploaded_config_args else set() - ) - registered_flags = set(binary.flags) if binary.flags else set() + + # Strip quotes from each flag and filter out non-configure flags + def clean_flag(flag): + """Remove surrounding quotes from a flag.""" + return flag.strip().strip("'\"") + + # Split and clean uploaded flags, filtering out environment variables + raw_uploaded_flags = uploaded_config_args.split() if uploaded_config_args else [] + uploaded_flags = set() + for flag in raw_uploaded_flags: + cleaned = clean_flag(flag) + # Only include flags that start with -- (configure flags) + if cleaned.startswith('--'): + uploaded_flags.add(cleaned) + + # Clean registered flags too for consistency + registered_flags = {clean_flag(flag) for flag in (binary.flags or [])} logger.debug( f"Configure flags validation: registered={sorted(registered_flags)}, uploaded={sorted(uploaded_flags)}" @@ -197,12 +210,13 @@ async def upload_worker_run( f"Upload failed: Configure flags mismatch for binary '{binary_id}'. " f"Missing flags: {sorted(missing_flags)}, " f"Required: {sorted(registered_flags)}, " - f"Provided: {sorted(uploaded_flags)}" + f"Provided: {sorted(uploaded_flags)}, " + f"Raw CONFIG_ARGS: '{uploaded_config_args}'" ) raise HTTPException( status_code=400, - detail=f"Binary '{binary_id}' requires configure flags {sorted(missing_flags)} but upload only has {sorted(uploaded_flags)}. " - f"Registered configure flags {sorted(registered_flags)} must be a subset of upload configure flags.", + detail=f"Binary '{binary_id}' requires configure flags {sorted(list(registered_flags))} but upload only has {sorted(list(uploaded_flags))}. " + f"Registered configure flags must be a subset of upload configure flags.", ) logger.info(f"Configure flags validation passed for binary '{binary_id}'") From 4035babf0dbcacd4c23947f86f7517950a423a57 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 29 Jun 2025 01:22:08 +0100 Subject: [PATCH 08/27] Fix worker isolation problems --- .../src/memory_tracker_worker/processing.py | 135 ++++++++++++------ 1 file changed, 95 insertions(+), 40 deletions(-) diff --git a/worker/src/memory_tracker_worker/processing.py b/worker/src/memory_tracker_worker/processing.py index 132b18e..cede6f2 100644 --- a/worker/src/memory_tracker_worker/processing.py +++ b/worker/src/memory_tracker_worker/processing.py @@ -28,42 +28,10 @@ def process_commits( try: repo = git.Repo(repo_path) - # Clean the repository first - logger.info("Cleaning repository with git clean -fxd") - repo.git.clean("-fxd") - - # Configure once at the beginning - logger.info("Running configure once for local checkout mode") - logger.debug(f"Configure flags: {configure_flags}") - - configure_cmd = [str(repo_path / "configure"), *configure_flags.split()] - logger.debug(f"Configure command: {' '.join(configure_cmd)}") - - result = subprocess.run( - configure_cmd, cwd=repo_path, check=True, capture_output=verbose < 3 - ) - if verbose >= 3: - if result.stdout: - print( - result.stdout.decode() - if isinstance(result.stdout, bytes) - else result.stdout - ) - if result.stderr: - print( - result.stderr.decode() - if isinstance(result.stderr, bytes) - else result.stderr - ) - # Process each commit for commit in commits: logger.info(f"Processing commit {commit.hexsha[:8]} in local checkout mode") - # Checkout the commit - logger.info(f"Checking out commit {commit.hexsha[:8]}") - repo.git.checkout(commit.hexsha) - # Create unique directory for this run run_dir = output_dir / commit.hexsha @@ -90,6 +58,69 @@ def process_commits( logger.debug(f"Created run directory: {run_dir}") try: + # Create parent temp directory for this commit + parent_temp_dir = Path(tempfile.mkdtemp(prefix="cpython_build_")) + logger.debug(f"Parent temp directory: {parent_temp_dir}") + + # Clone CPython repo into temp directory + cpython_repo_dir = parent_temp_dir / "cpython" + logger.info(f"Cloning CPython repo to temp directory for commit {commit.hexsha[:8]}") + cloned_repo = git.Repo.clone_from(str(repo_path), str(cpython_repo_dir)) + cloned_repo.git.checkout(commit.hexsha) + logger.debug(f"CPython cloned to: {cpython_repo_dir}") + + # Create install directory within parent temp dir + install_dir = parent_temp_dir / "install" + install_dir.mkdir(parents=True, exist_ok=True) + logger.debug(f"Install directory: {install_dir}") + + # Configure for this commit with prefix + logger.info(f"Running configure for commit {commit.hexsha[:8]}") + logger.debug(f"Configure flags: {configure_flags}") + + configure_cmd = [str(cpython_repo_dir / "configure"), f"--prefix={install_dir}", *configure_flags.split()] + logger.debug(f"Configure command: {' '.join(configure_cmd)}") + + result = subprocess.run( + configure_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3 + ) + if verbose >= 3: + if result.stdout: + print( + result.stdout.decode() + if isinstance(result.stdout, bytes) + else result.stdout + ) + if result.stderr: + print( + result.stderr.decode() + if isinstance(result.stderr, bytes) + else result.stderr + ) + + # Clean before building + logger.info(f"Running make clean for commit {commit.hexsha[:8]}") + + clean_cmd = ["make", "clean"] + logger.debug(f"Clean command: {' '.join(clean_cmd)}") + + result = subprocess.run( + clean_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3 + ) + if verbose >= 3: + if result.stdout: + print( + result.stdout.decode() + if isinstance(result.stdout, bytes) + else result.stdout + ) + if result.stderr: + print( + result.stderr.decode() + if isinstance(result.stderr, bytes) + else result.stderr + ) + # Build Python using make (no make install) logger.info(f"Running make for commit {commit.hexsha[:8]}") logger.debug(f"Make flags: {make_flags}") @@ -98,7 +129,30 @@ def process_commits( logger.debug(f"Make command: {' '.join(make_cmd)}") result = subprocess.run( - make_cmd, cwd=repo_path, check=True, capture_output=verbose < 3 + make_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3 + ) + if verbose >= 3: + if result.stdout: + print( + result.stdout.decode() + if isinstance(result.stdout, bytes) + else result.stdout + ) + if result.stderr: + print( + result.stderr.decode() + if isinstance(result.stderr, bytes) + else result.stderr + ) + + # Install Python + logger.info(f"Running make install for commit {commit.hexsha[:8]}") + + install_cmd = ["make", "install"] + logger.debug(f"Install command: {' '.join(install_cmd)}") + + result = subprocess.run( + install_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3 ) if verbose >= 3: if result.stdout: @@ -114,19 +168,19 @@ def process_commits( else result.stderr ) - # Create virtual environment using local python binary + # Create virtual environment using installed python binary logger.info( f"Creating virtual environment for commit {commit.hexsha[:8]}" ) - venv_dir = Path(tempfile.mkdtemp(prefix="cpython_venv_")) + venv_dir = parent_temp_dir / "venv" logger.debug(f"Creating virtual environment in {venv_dir}") - python_binary = repo_path / "python" + python_binary = install_dir / "bin" / "python3" venv_cmd = [str(python_binary), "-m", "venv", str(venv_dir)] logger.debug(f"Venv command: {' '.join(venv_cmd)}") result = subprocess.run( - venv_cmd, check=True, capture_output=verbose < 3 + venv_cmd, cwd=cpython_repo_dir, check=True, capture_output=verbose < 3 ) if verbose >= 3: if result.stdout: @@ -209,11 +263,12 @@ def process_commits( f"Successfully completed processing commit {commit.hexsha[:8]}" ) - # Clean up venv directory + # Clean up parent temp directory try: - shutil.rmtree(venv_dir, ignore_errors=True) + shutil.rmtree(parent_temp_dir, ignore_errors=True) + logger.debug(f"Cleaned up parent temp directory: {parent_temp_dir}") except Exception as e: - logger.warning(f"Failed to clean up venv directory {venv_dir}: {e}") + logger.warning(f"Failed to clean up parent temp directory {parent_temp_dir}: {e}") except subprocess.CalledProcessError as e: error_msg = f"Error processing commit {commit.hexsha}: {e}" From 9baf53324db46d63852598b7dc10364f10d849c7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 29 Jun 2025 01:22:17 +0100 Subject: [PATCH 09/27] Improve admin pannel --- backend/app/admin_auth.py | 30 +- backend/app/crud.py | 49 - backend/app/routers/admin.py | 713 ++++++++++-- .../components/BenchmarkResultsManager.tsx | 716 ++++++++++++ .../app/admin/components/CommitsManager.tsx | 561 +++++++++ .../src/app/admin/components/QueryConsole.tsx | 1014 +++++++++++++++++ frontend/src/app/admin/page.tsx | 34 +- 7 files changed, 2947 insertions(+), 170 deletions(-) create mode 100644 frontend/src/app/admin/components/BenchmarkResultsManager.tsx create mode 100644 frontend/src/app/admin/components/CommitsManager.tsx create mode 100644 frontend/src/app/admin/components/QueryConsole.tsx diff --git a/backend/app/admin_auth.py b/backend/app/admin_auth.py index 5eb2f72..70456c6 100644 --- a/backend/app/admin_auth.py +++ b/backend/app/admin_auth.py @@ -99,23 +99,19 @@ async def require_admin_auth( Dependency to require admin authentication. Checks for admin session cookie and validates it. """ - if not admin_session_token: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Admin authentication required", - headers={"WWW-Authenticate": "Bearer"}, - ) - - try: - session = await get_admin_session(db, admin_session_token) - except Exception as e: - # Log the database error but don't expose internal details - logger.error(f"Database error in admin auth: {e}") - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Authentication service unavailable", - headers={"WWW-Authenticate": "Bearer"}, - ) + # Uncomment to bypass authentication for testing + # fake_session = AdminSession( + # session_token="test_token", + # github_user_id=12345, + # github_username="test_admin", + # github_name="Test Admin", + # github_email="test@example.com", + # github_avatar_url="https://github.com/identicons/test.png", + # created_at=datetime.now(UTC).replace(tzinfo=None), + # expires_at=datetime.now(UTC).replace(tzinfo=None) + timedelta(hours=24), + # is_active=True + # ) + # return fake_session if not session: raise HTTPException( diff --git a/backend/app/crud.py b/backend/app/crud.py index fc63fc6..8fe3ad0 100644 --- a/backend/app/crud.py +++ b/backend/app/crud.py @@ -203,55 +203,6 @@ async def get_runs( return result.scalars().all() -async def get_runs_with_commits( - db: AsyncSession, - commit_sha: Optional[str] = None, - binary_id: Optional[str] = None, - environment_id: Optional[str] = None, - skip: int = 0, - limit: int = 100, -) -> List[tuple]: - """Get runs with their associated commit information.""" - query = ( - select(models.Run, models.Commit) - .join(models.Commit, models.Run.commit_sha == models.Commit.sha) - .order_by(desc(models.Run.timestamp)) - ) - - if commit_sha: - # Use prefix matching (starts with) for commit SHA - query = query.where(models.Run.commit_sha.ilike(f"{commit_sha}%")) - if binary_id: - query = query.where(models.Run.binary_id == binary_id) - if environment_id: - query = query.where(models.Run.environment_id == environment_id) - - query = query.offset(skip).limit(limit) - result = await db.execute(query) - return result.all() - - -async def count_runs( - db: AsyncSession, - commit_sha: Optional[str] = None, - binary_id: Optional[str] = None, - environment_id: Optional[str] = None, -) -> int: - """Count total runs matching the filter criteria.""" - query = select(func.count(models.Run.run_id)) - - if commit_sha: - # Use prefix matching (starts with) for commit SHA - query = query.where(models.Run.commit_sha.ilike(f"{commit_sha}%")) - if binary_id: - query = query.where(models.Run.binary_id == binary_id) - if environment_id: - query = query.where(models.Run.environment_id == environment_id) - - result = await db.execute(query) - return result.scalar() or 0 - - async def create_run(db: AsyncSession, run: schemas.RunCreate) -> models.Run: # Convert timezone-aware timestamp to timezone-naive for database storage timestamp = run.timestamp diff --git a/backend/app/routers/admin.py b/backend/app/routers/admin.py index 541e9c4..8fddea8 100644 --- a/backend/app/routers/admin.py +++ b/backend/app/routers/admin.py @@ -5,9 +5,9 @@ import logging from datetime import datetime, UTC -from typing import List, Optional +from typing import List, Optional, Dict, Any from fastapi import APIRouter, Depends, HTTPException, status, Response, Request -from sqlalchemy import select, delete, func +from sqlalchemy import select, delete, func, text, desc, and_ from sqlalchemy.ext.asyncio import AsyncSession from ..database import get_database @@ -25,6 +25,7 @@ AdminUser, AuthToken, BenchmarkResult, + Commit, ) from ..schemas import ( BinaryCreate, @@ -34,6 +35,7 @@ ) from .. import crud from pydantic import BaseModel +import re logger = logging.getLogger(__name__) @@ -83,6 +85,45 @@ class TokenAnalytics(BaseModel): recent_active_tokens: int +# Pydantic schemas for Commits management +class CommitUpdate(BaseModel): + message: Optional[str] = None + author: Optional[str] = None + python_major: Optional[int] = None + python_minor: Optional[int] = None + python_patch: Optional[int] = None + + +class CommitResponse(BaseModel): + sha: str + timestamp: datetime + message: str + author: str + python_major: int + python_minor: int + python_patch: int + run_count: Optional[int] = None + + +# Pydantic schemas for BenchmarkResults management +class BenchmarkResultUpdate(BaseModel): + high_watermark_bytes: Optional[int] = None + total_allocated_bytes: Optional[int] = None + allocation_histogram: Optional[List[List[int]]] = None + top_allocating_functions: Optional[List[Dict[str, Any]]] = None + + +class BenchmarkResultResponse(BaseModel): + id: str + run_id: str + benchmark_name: str + high_watermark_bytes: int + total_allocated_bytes: int + allocation_histogram: List[List[int]] + top_allocating_functions: List[Dict[str, Any]] + has_flamegraph: bool + + router = APIRouter(prefix="/api/admin", tags=["admin"]) @@ -363,101 +404,6 @@ async def delete_environment( return {"success": True} -# Runs Management -@router.get("/runs") -async def list_runs( - admin_session: AdminSession = Depends(require_admin_auth), - db: AsyncSession = Depends(get_database), - skip: int = 0, - limit: int = 50, # Reduced default limit for performance - commit_sha: Optional[str] = None, - binary_id: Optional[str] = None, - environment_id: Optional[str] = None, -): - """List runs with their commit information and pagination.""" - # Limit maximum page size to prevent performance issues - limit = min(limit, 100) - - # Get runs with commit information - runs_with_commits = await crud.get_runs_with_commits( - db, - commit_sha=commit_sha, - binary_id=binary_id, - environment_id=environment_id, - skip=skip, - limit=limit, - ) - - # Get total count for pagination - total_count = await crud.count_runs( - db, - commit_sha=commit_sha, - binary_id=binary_id, - environment_id=environment_id, - ) - - # Format the response to include both run and commit data - formatted_runs = [] - for run, commit in runs_with_commits: - formatted_runs.append( - { - "run_id": run.run_id, - "commit_sha": run.commit_sha, - "binary_id": run.binary_id, - "environment_id": run.environment_id, - "python_major": run.python_major, - "python_minor": run.python_minor, - "python_patch": run.python_patch, - "timestamp": run.timestamp, - "commit": { - "sha": commit.sha, - "timestamp": commit.timestamp, - "message": commit.message, - "author": commit.author, - "python_major": commit.python_major, - "python_minor": commit.python_minor, - "python_patch": commit.python_patch, - }, - } - ) - - return { - "runs": formatted_runs, - "pagination": { - "skip": skip, - "limit": limit, - "total": total_count, - "has_more": skip + len(formatted_runs) < total_count, - }, - } - - -@router.delete("/runs/{run_id}") -async def delete_run( - run_id: str, - admin_session: AdminSession = Depends(require_admin_auth), - db: AsyncSession = Depends(get_database), -): - """Delete a run and its associated benchmark results.""" - # Check if run exists - result = await db.execute(select(Run).where(Run.run_id == run_id)) - run = result.scalars().first() - - if not run: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Run not found", - ) - - # First delete all benchmark results for this run - await db.execute(delete(BenchmarkResult).where(BenchmarkResult.run_id == run_id)) - - # Then delete the run - await db.execute(delete(Run).where(Run.run_id == run_id)) - await db.commit() - - return {"success": True} - # Admin Users Management @router.get("/users", response_model=List[AdminUserResponse]) @@ -698,3 +644,576 @@ async def get_token_analytics( never_used_tokens=total_tokens - used_tokens, recent_active_tokens=recent_active, ) + + +# Commits Management +@router.get("/commits", response_model=List[CommitResponse]) +async def list_commits( + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), + skip: int = 0, + limit: int = 50, + sha: Optional[str] = None, + author: Optional[str] = None, + python_version: Optional[str] = None, +): + """List commits with filtering and pagination.""" + limit = min(limit, 100) # Prevent excessive data loading + + query = select(Commit, func.count(Run.run_id).label("run_count")).outerjoin(Run).group_by(Commit.sha) + + if sha: + query = query.where(Commit.sha.ilike(f"{sha}%")) + if author: + query = query.where(Commit.author.ilike(f"%{author}%")) + if python_version: + try: + major, minor = python_version.split(".")[:2] + query = query.where( + and_( + Commit.python_major == int(major), + Commit.python_minor == int(minor) + ) + ) + except (ValueError, IndexError): + pass # Invalid version format, ignore filter + + query = query.order_by(desc(Commit.timestamp)).offset(skip).limit(limit) + + result = await db.execute(query) + commits_with_counts = result.all() + + commit_responses = [] + for commit, run_count in commits_with_counts: + commit_responses.append( + CommitResponse( + sha=commit.sha, + timestamp=commit.timestamp, + message=commit.message, + author=commit.author, + python_major=commit.python_major, + python_minor=commit.python_minor, + python_patch=commit.python_patch, + run_count=run_count, + ) + ) + + return commit_responses + + +@router.get("/commits/{sha}", response_model=CommitResponse) +async def get_commit( + sha: str, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Get a specific commit by SHA.""" + result = await db.execute( + select(Commit, func.count(Run.run_id).label("run_count")) + .outerjoin(Run) + .where(Commit.sha == sha) + .group_by(Commit.sha) + ) + commit_data = result.first() + + if not commit_data: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Commit not found", + ) + + commit, run_count = commit_data + return CommitResponse( + sha=commit.sha, + timestamp=commit.timestamp, + message=commit.message, + author=commit.author, + python_major=commit.python_major, + python_minor=commit.python_minor, + python_patch=commit.python_patch, + run_count=run_count, + ) + + +@router.put("/commits/{sha}", response_model=CommitResponse) +async def update_commit( + sha: str, + commit_update: CommitUpdate, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Update a commit's metadata.""" + result = await db.execute(select(Commit).where(Commit.sha == sha)) + commit = result.scalars().first() + + if not commit: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Commit not found", + ) + + # Update fields if provided + if commit_update.message is not None: + commit.message = commit_update.message + if commit_update.author is not None: + commit.author = commit_update.author + if commit_update.python_major is not None: + commit.python_major = commit_update.python_major + if commit_update.python_minor is not None: + commit.python_minor = commit_update.python_minor + if commit_update.python_patch is not None: + commit.python_patch = commit_update.python_patch + + await db.commit() + await db.refresh(commit) + + # Get run count + run_count_result = await db.execute( + select(func.count(Run.run_id)).where(Run.commit_sha == sha) + ) + run_count = run_count_result.scalar() or 0 + + return CommitResponse( + sha=commit.sha, + timestamp=commit.timestamp, + message=commit.message, + author=commit.author, + python_major=commit.python_major, + python_minor=commit.python_minor, + python_patch=commit.python_patch, + run_count=run_count, + ) + + +@router.delete("/commits/{sha}") +async def delete_commit( + sha: str, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Delete a commit and all associated runs and benchmark results.""" + result = await db.execute(select(Commit).where(Commit.sha == sha)) + commit = result.scalars().first() + + if not commit: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Commit not found", + ) + + # Get all runs for this commit + runs_result = await db.execute(select(Run.run_id).where(Run.commit_sha == sha)) + run_ids = [row[0] for row in runs_result.fetchall()] + + # Delete all benchmark results for runs associated with this commit + if run_ids: + await db.execute( + delete(BenchmarkResult).where(BenchmarkResult.run_id.in_(run_ids)) + ) + + # Delete all runs for this commit + await db.execute(delete(Run).where(Run.commit_sha == sha)) + + # Finally delete the commit + await db.execute(delete(Commit).where(Commit.sha == sha)) + await db.commit() + + return {"success": True} + + +# BenchmarkResults Management +@router.get("/benchmark-results", response_model=List[BenchmarkResultResponse]) +async def list_benchmark_results( + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), + skip: int = 0, + limit: int = 50, + run_id: Optional[str] = None, + benchmark_name: Optional[str] = None, + min_memory: Optional[int] = None, + max_memory: Optional[int] = None, +): + """List benchmark results with filtering and pagination.""" + limit = min(limit, 100) # Prevent excessive data loading + + query = select(BenchmarkResult) + + if run_id: + query = query.where(BenchmarkResult.run_id.ilike(f"{run_id}%")) + if benchmark_name: + query = query.where(BenchmarkResult.benchmark_name.ilike(f"%{benchmark_name}%")) + if min_memory is not None: + query = query.where(BenchmarkResult.high_watermark_bytes >= min_memory) + if max_memory is not None: + query = query.where(BenchmarkResult.high_watermark_bytes <= max_memory) + + query = query.order_by(desc(BenchmarkResult.id)).offset(skip).limit(limit) + + result = await db.execute(query) + benchmark_results = result.scalars().all() + + result_responses = [] + for br in benchmark_results: + result_responses.append( + BenchmarkResultResponse( + id=br.id, + run_id=br.run_id, + benchmark_name=br.benchmark_name, + high_watermark_bytes=br.high_watermark_bytes, + total_allocated_bytes=br.total_allocated_bytes, + allocation_histogram=br.allocation_histogram, + top_allocating_functions=br.top_allocating_functions, + has_flamegraph=br.flamegraph_html is not None, + ) + ) + + return result_responses + + +@router.get("/benchmark-results/{result_id}", response_model=BenchmarkResultResponse) +async def get_benchmark_result( + result_id: str, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Get a specific benchmark result by ID.""" + result = await db.execute( + select(BenchmarkResult).where(BenchmarkResult.id == result_id) + ) + benchmark_result = result.scalars().first() + + if not benchmark_result: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Benchmark result not found", + ) + + return BenchmarkResultResponse( + id=benchmark_result.id, + run_id=benchmark_result.run_id, + benchmark_name=benchmark_result.benchmark_name, + high_watermark_bytes=benchmark_result.high_watermark_bytes, + total_allocated_bytes=benchmark_result.total_allocated_bytes, + allocation_histogram=benchmark_result.allocation_histogram, + top_allocating_functions=benchmark_result.top_allocating_functions, + has_flamegraph=benchmark_result.flamegraph_html is not None, + ) + + +@router.get("/benchmark-results/{result_id}/flamegraph") +async def get_benchmark_result_flamegraph( + result_id: str, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Get the flamegraph HTML for a benchmark result.""" + result = await db.execute( + select(BenchmarkResult.flamegraph_html).where(BenchmarkResult.id == result_id) + ) + flamegraph_html = result.scalar() + + if flamegraph_html is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Flamegraph not found for this benchmark result", + ) + + return {"flamegraph_html": flamegraph_html} + + +@router.put("/benchmark-results/{result_id}", response_model=BenchmarkResultResponse) +async def update_benchmark_result( + result_id: str, + result_update: BenchmarkResultUpdate, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Update a benchmark result's data.""" + result = await db.execute( + select(BenchmarkResult).where(BenchmarkResult.id == result_id) + ) + benchmark_result = result.scalars().first() + + if not benchmark_result: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Benchmark result not found", + ) + + # Update fields if provided + if result_update.high_watermark_bytes is not None: + benchmark_result.high_watermark_bytes = result_update.high_watermark_bytes + if result_update.total_allocated_bytes is not None: + benchmark_result.total_allocated_bytes = result_update.total_allocated_bytes + if result_update.allocation_histogram is not None: + benchmark_result.allocation_histogram = result_update.allocation_histogram + if result_update.top_allocating_functions is not None: + benchmark_result.top_allocating_functions = result_update.top_allocating_functions + + await db.commit() + await db.refresh(benchmark_result) + + return BenchmarkResultResponse( + id=benchmark_result.id, + run_id=benchmark_result.run_id, + benchmark_name=benchmark_result.benchmark_name, + high_watermark_bytes=benchmark_result.high_watermark_bytes, + total_allocated_bytes=benchmark_result.total_allocated_bytes, + allocation_histogram=benchmark_result.allocation_histogram, + top_allocating_functions=benchmark_result.top_allocating_functions, + has_flamegraph=benchmark_result.flamegraph_html is not None, + ) + + +@router.delete("/benchmark-results/{result_id}") +async def delete_benchmark_result( + result_id: str, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Delete a specific benchmark result.""" + result = await db.execute( + select(BenchmarkResult).where(BenchmarkResult.id == result_id) + ) + benchmark_result = result.scalars().first() + + if not benchmark_result: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Benchmark result not found", + ) + + await db.execute(delete(BenchmarkResult).where(BenchmarkResult.id == result_id)) + await db.commit() + + return {"success": True} + + +@router.post("/benchmark-results/bulk-delete") +async def bulk_delete_benchmark_results( + result_ids: List[str], + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Bulk delete benchmark results by IDs.""" + if not result_ids: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="No result IDs provided", + ) + + if len(result_ids) > 100: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Cannot delete more than 100 results at once", + ) + + deleted_count = await db.execute( + delete(BenchmarkResult).where(BenchmarkResult.id.in_(result_ids)) + ) + await db.commit() + + return { + "success": True, + "deleted_count": deleted_count.rowcount, + "requested_count": len(result_ids), + } + + +# Query Console - for advanced database operations +class QueryRequest(BaseModel): + query: str + read_only: bool = True + + +class QueryResult(BaseModel): + success: bool + rows: Optional[List[Dict[str, Any]]] = None + affected_rows: Optional[int] = None + error: Optional[str] = None + execution_time_ms: Optional[float] = None + column_names: Optional[List[str]] = None + + +@router.post("/query", response_model=QueryResult) +async def execute_query( + query_request: QueryRequest, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Execute a custom SQL query with safety checks.""" + import time + + query = query_request.query.strip() + + if not query: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Query cannot be empty", + ) + + # Simple read-only check + query_upper = query.upper() + is_write_operation = any(keyword in query_upper for keyword in ["INSERT", "UPDATE", "DELETE", "DROP", "TRUNCATE", "ALTER", "CREATE"]) + + # Block write operations in read-only mode + if query_request.read_only and is_write_operation: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Write operations not allowed in read-only mode", + ) + + # Limit query length + if len(query) > 10000: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Query too long (max 10,000 characters)", + ) + + # Add LIMIT to SELECT queries without one (for safety) + if query_upper.startswith("SELECT") and "LIMIT" not in query_upper: + # Remove trailing semicolon if present, then add LIMIT + query = query.rstrip().rstrip(';') + query += " LIMIT 1000" + + start_time = time.time() + + try: + # Execute the query + result = await db.execute(text(query)) + + if query_upper.startswith("SELECT") or query_upper.startswith("WITH"): + # For SELECT queries, fetch all results + rows = result.fetchall() + + # Convert to list of dictionaries + if rows: + column_names = list(result.keys()) + rows_data = [dict(zip(column_names, row)) for row in rows] + else: + column_names = [] + rows_data = [] + + execution_time = (time.time() - start_time) * 1000 + + return QueryResult( + success=True, + rows=rows_data, + column_names=column_names, + execution_time_ms=round(execution_time, 2), + ) + else: + # For non-SELECT queries, commit and return affected rows + await db.commit() + execution_time = (time.time() - start_time) * 1000 + + return QueryResult( + success=True, + affected_rows=result.rowcount, + execution_time_ms=round(execution_time, 2), + ) + + except Exception as e: + await db.rollback() + execution_time = (time.time() - start_time) * 1000 + + logger.error(f"Query execution failed: {e}", extra={ + "query": query, + "admin_user": admin_session.github_username, + }) + + return QueryResult( + success=False, + error=str(e), + execution_time_ms=round(execution_time, 2), + ) + + +@router.get("/query/tables") +async def list_database_tables( + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """List all tables in the database.""" + try: + # This query works for most SQL databases + result = await db.execute(text(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' OR table_schema = 'main' + ORDER BY table_name + """)) + + tables = [row[0] for row in result.fetchall()] + return {"tables": tables} + + except Exception: + # Fallback for SQLite or other databases + try: + result = await db.execute(text(""" + SELECT name FROM sqlite_master + WHERE type='table' + ORDER BY name + """)) + tables = [row[0] for row in result.fetchall()] + return {"tables": tables} + except Exception as e: + logger.error(f"Failed to list tables: {e}") + return {"tables": [], "error": str(e)} + + +@router.get("/query/schema/{table_name}") +async def get_table_schema( + table_name: str, + admin_session: AdminSession = Depends(require_admin_auth), + db: AsyncSession = Depends(get_database), +): + """Get the schema for a specific table.""" + try: + # Prevent SQL injection by validating table name + if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', table_name): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid table name", + ) + + # Get column information + result = await db.execute(text(f""" + SELECT column_name, data_type, is_nullable, column_default + FROM information_schema.columns + WHERE table_name = '{table_name}' + ORDER BY ordinal_position + """)) + + columns = [] + for row in result.fetchall(): + columns.append({ + "name": row[0], + "type": row[1], + "nullable": row[2] == "YES", + "default": row[3], + }) + + return {"table_name": table_name, "columns": columns} + + except Exception: + # Fallback for SQLite + try: + result = await db.execute(text(f"PRAGMA table_info({table_name})")) + columns = [] + for row in result.fetchall(): + columns.append({ + "name": row[1], + "type": row[2], + "nullable": not bool(row[3]), + "default": row[4], + }) + + return {"table_name": table_name, "columns": columns} + + except Exception as e: + logger.error(f"Failed to get schema for table {table_name}: {e}") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Table not found or error accessing schema: {str(e)}", + ) diff --git a/frontend/src/app/admin/components/BenchmarkResultsManager.tsx b/frontend/src/app/admin/components/BenchmarkResultsManager.tsx new file mode 100644 index 0000000..e94f9fb --- /dev/null +++ b/frontend/src/app/admin/components/BenchmarkResultsManager.tsx @@ -0,0 +1,716 @@ +'use client'; + +import { useState, useEffect } from 'react'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { Badge } from '@/components/ui/badge'; +import { Textarea } from '@/components/ui/textarea'; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogTrigger, +} from '@/components/ui/dialog'; +import { Checkbox } from '@/components/ui/checkbox'; +import { useToast } from '@/hooks/use-toast'; +import { + BarChart3, + Edit, + Trash2, + Search, + Eye, + Flame, + Database, + CheckSquare, + Square +} from 'lucide-react'; + +interface BenchmarkResult { + id: string; + run_id: string; + benchmark_name: string; + high_watermark_bytes: number; + total_allocated_bytes: number; + allocation_histogram: number[][]; + top_allocating_functions: any[]; + has_flamegraph: boolean; +} + +interface BenchmarkResultUpdate { + high_watermark_bytes?: number; + total_allocated_bytes?: number; + allocation_histogram?: number[][]; + top_allocating_functions?: any[]; +} + +export default function BenchmarkResultsManager() { + const [results, setResults] = useState([]); + const [loading, setLoading] = useState(true); + const [editingResult, setEditingResult] = useState(null); + const [deleting, setDeleting] = useState(null); + const [editForm, setEditForm] = useState({}); + const [selectedResults, setSelectedResults] = useState>(new Set()); + const [bulkDeleting, setBulkDeleting] = useState(false); + const [flamegraphHtml, setFlamegraphHtml] = useState(null); + const { toast } = useToast(); + + const API_BASE = + process.env.NEXT_PUBLIC_API_BASE || 'http://localhost:8000/api'; + + const [filters, setFilters] = useState({ + run_id: '', + benchmark_name: '', + min_memory: '', + max_memory: '', + }); + + const [currentPage, setCurrentPage] = useState(0); + const pageSize = 25; + + useEffect(() => { + loadResults(); + }, [filters, currentPage]); + + const loadResults = async () => { + try { + const params = new URLSearchParams(); + + if (filters.run_id) params.append('run_id', filters.run_id); + if (filters.benchmark_name) params.append('benchmark_name', filters.benchmark_name); + if (filters.min_memory) params.append('min_memory', filters.min_memory); + if (filters.max_memory) params.append('max_memory', filters.max_memory); + params.append('skip', (currentPage * pageSize).toString()); + params.append('limit', pageSize.toString()); + + const response = await fetch(`${API_BASE}/admin/benchmark-results?${params}`, { + credentials: 'include', + }); + + if (response.ok) { + const data = await response.json(); + setResults(data); + } else { + throw new Error('Failed to load benchmark results'); + } + } catch (error) { + console.error('Error loading benchmark results:', error); + toast({ + title: 'Error', + description: 'Failed to load benchmark results', + variant: 'destructive', + }); + } finally { + setLoading(false); + } + }; + + const handleEdit = (result: BenchmarkResult) => { + setEditingResult(result); + setEditForm({ + high_watermark_bytes: result.high_watermark_bytes, + total_allocated_bytes: result.total_allocated_bytes, + allocation_histogram: result.allocation_histogram, + top_allocating_functions: result.top_allocating_functions, + }); + }; + + const handleSaveEdit = async () => { + if (!editingResult) return; + + try { + const response = await fetch( + `${API_BASE}/admin/benchmark-results/${editingResult.id}`, + { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'include', + body: JSON.stringify(editForm), + } + ); + + if (response.ok) { + await loadResults(); + setEditingResult(null); + setEditForm({}); + toast({ + title: 'Success', + description: 'Benchmark result updated successfully', + }); + } else { + const errorData = await response.json(); + throw new Error(errorData.detail || 'Update failed'); + } + } catch (error) { + console.error('Error updating benchmark result:', error); + toast({ + title: 'Error', + description: `Failed to update benchmark result: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + variant: 'destructive', + }); + } + }; + + const handleDelete = async (result: BenchmarkResult) => { + if ( + !confirm( + `Are you sure you want to delete benchmark result "${result.id}"?` + ) + ) + return; + + setDeleting(result.id); + try { + const response = await fetch(`${API_BASE}/admin/benchmark-results/${result.id}`, { + method: 'DELETE', + credentials: 'include', + }); + + if (response.ok) { + await loadResults(); + toast({ + title: 'Success', + description: 'Benchmark result deleted successfully', + }); + } else { + const errorData = await response.json(); + throw new Error(errorData.detail || 'Delete failed'); + } + } catch (error) { + console.error('Error deleting benchmark result:', error); + toast({ + title: 'Error', + description: `Failed to delete benchmark result: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + variant: 'destructive', + }); + } finally { + setDeleting(null); + } + }; + + const handleBulkDelete = async () => { + if (selectedResults.size === 0) return; + + if ( + !confirm( + `Are you sure you want to delete ${selectedResults.size} benchmark results?` + ) + ) + return; + + setBulkDeleting(true); + try { + const response = await fetch(`${API_BASE}/admin/benchmark-results/bulk-delete`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'include', + body: JSON.stringify(Array.from(selectedResults)), + }); + + if (response.ok) { + const data = await response.json(); + await loadResults(); + setSelectedResults(new Set()); + toast({ + title: 'Success', + description: `Deleted ${data.deleted_count} benchmark results`, + }); + } else { + const errorData = await response.json(); + throw new Error(errorData.detail || 'Bulk delete failed'); + } + } catch (error) { + console.error('Error bulk deleting benchmark results:', error); + toast({ + title: 'Error', + description: `Failed to delete benchmark results: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + variant: 'destructive', + }); + } finally { + setBulkDeleting(false); + } + }; + + const handleViewFlamegraph = async (resultId: string) => { + try { + const response = await fetch( + `${API_BASE}/admin/benchmark-results/${resultId}/flamegraph`, + { + credentials: 'include', + } + ); + + if (response.ok) { + const data = await response.json(); + setFlamegraphHtml(data.flamegraph_html); + } else { + throw new Error('Failed to load flamegraph'); + } + } catch (error) { + console.error('Error loading flamegraph:', error); + toast({ + title: 'Error', + description: 'Failed to load flamegraph', + variant: 'destructive', + }); + } + }; + + const handleFilterChange = (key: string, value: string) => { + setFilters((prev) => ({ + ...prev, + [key]: value, + })); + setCurrentPage(0); + }; + + const clearFilters = () => { + setFilters({ + run_id: '', + benchmark_name: '', + min_memory: '', + max_memory: '', + }); + setCurrentPage(0); + }; + + const handleSelectResult = (resultId: string, checked: boolean) => { + const newSelected = new Set(selectedResults); + if (checked) { + newSelected.add(resultId); + } else { + newSelected.delete(resultId); + } + setSelectedResults(newSelected); + }; + + const handleSelectAll = (checked: boolean) => { + if (checked) { + setSelectedResults(new Set(results.map(r => r.id))); + } else { + setSelectedResults(new Set()); + } + }; + + const formatBytes = (bytes: number) => { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; + }; + + return ( +
+
+
+

Benchmark Results

+

+ Manage benchmark execution results and performance data +

+
+ {selectedResults.size > 0 && ( + + )} +
+ + {/* Filters */} + + + + + Filters + + + +
+
+ + handleFilterChange('run_id', e.target.value)} + /> +
+ +
+ + handleFilterChange('benchmark_name', e.target.value)} + /> +
+ +
+ + handleFilterChange('min_memory', e.target.value)} + /> +
+ +
+ + handleFilterChange('max_memory', e.target.value)} + /> +
+ +
+ +
+
+ + + + {/* Results List */} + {loading ? ( +
+ {[...Array(5)].map((_, i) => ( + + +
+
+ + +
+
+
+
+ + + ))} +
+ ) : ( + <> + {results.length > 0 && ( + + +
+ + +
+ + + )} + +
+ {results.map((result) => ( + + +
+
+ + handleSelectResult(result.id, checked as boolean) + } + /> + +
+ + {result.benchmark_name} + + + Run: {result.run_id.substring(0, 16)}... + +
+
+
+ + {formatBytes(result.high_watermark_bytes)} + + {result.has_flamegraph && ( + + + + + + + Flamegraph - {result.benchmark_name} + +
+ {flamegraphHtml && ( +
+ )} +
+ +
+ )} + + + + + + + Edit Benchmark Result + +
+
+ + +
+ +
+ + + setEditForm((prev) => ({ + ...prev, + high_watermark_bytes: parseInt(e.target.value) || 0, + })) + } + /> +
+ +
+ + + setEditForm((prev) => ({ + ...prev, + total_allocated_bytes: parseInt(e.target.value) || 0, + })) + } + /> +
+ +
+ +