diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..016a1e9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Default review owner +* @xxg1413 diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..a60cc3f --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: pxiaoer +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/ISSUE_TEMPLATE/broken-link.yml b/.github/ISSUE_TEMPLATE/broken-link.yml new file mode 100644 index 0000000..2ca94a8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/broken-link.yml @@ -0,0 +1,66 @@ +name: Broken or moved link +description: Report a catalog URL that is permanently unavailable or has moved. +title: "[Broken link]: " +labels: + - broken-link +body: + - type: markdown + attributes: + value: | + A single 403, 429, timeout, or transient 5xx does not prove a resource is broken. Include repeatable evidence so maintainers can distinguish blocking and rate limiting from removal. + - type: input + id: page + attributes: + label: FlyPython page or catalog ID + placeholder: ai-agents / openai-agents-sdk + validations: + required: true + - type: input + id: url + attributes: + label: Affected URL + placeholder: https://example.org/old-page + validations: + required: true + - type: input + id: observed-on + attributes: + label: Observation date + description: Use YYYY-MM-DD. + placeholder: "2026-08-31" + validations: + required: true + - type: dropdown + id: result + attributes: + label: Observed result + options: + - 404 Not Found + - 410 Gone + - Redirects to unrelated content + - DNS or TLS failure + - 403 Forbidden + - 429 Too Many Requests + - Timeout or 5xx + - Other + validations: + required: true + - type: textarea + id: evidence + attributes: + label: Reproduction and evidence + description: Include what you tried, whether it reproduced, and any final redirect URL. Do not include cookies, tokens, or personal data. + validations: + required: true + - type: input + id: replacement + attributes: + label: Official replacement URL + description: Optional. Provide only a canonical replacement from the same publisher or project. + - type: checkboxes + id: confirmation + attributes: + label: Confirmation + options: + - label: I checked that this report does not expose a credential or security vulnerability. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..297a377 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Python learning questions and discussion + url: https://github.com/flypythoncom/python/discussions + about: Ask for learning advice, compare resources, or start a general discussion. + - name: Private security report + url: https://github.com/flypythoncom/python/security/advisories/new + about: Report vulnerabilities privately. Do not disclose them in a public issue. diff --git a/.github/ISSUE_TEMPLATE/resource-proposal.yml b/.github/ISSUE_TEMPLATE/resource-proposal.yml new file mode 100644 index 0000000..4e16d2e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/resource-proposal.yml @@ -0,0 +1,99 @@ +name: Resource proposal +description: Propose an official Python or AI-agent resource for the reviewed catalog. +title: "[Resource]: " +labels: + - resource-proposal +body: + - type: markdown + attributes: + value: | + Thanks for helping improve FlyPython. Read the [curation policy](https://github.com/flypythoncom/python/blob/master/docs/CURATION_POLICY.md) before submitting. General learning questions belong in Discussions. + - type: input + id: title + attributes: + label: Resource name + placeholder: Python Packaging User Guide + validations: + required: true + - type: input + id: url + attributes: + label: Canonical URL + description: Use the official publisher, standard, documentation, or project URL. + placeholder: https://packaging.python.org/ + validations: + required: true + - type: dropdown + id: path + attributes: + label: Learning path + options: + - foundations + - web-apis + - automation + - ai-agents + validations: + required: true + - type: dropdown + id: source-type + attributes: + label: Source type + options: + - official-docs + - official-standard + - official-project + validations: + required: true + - type: dropdown + id: level + attributes: + label: Learner level + options: + - beginner + - intermediate + - advanced + - all-levels + validations: + required: true + - type: dropdown + id: language + attributes: + label: Resource language + description: Select the catalog's machine-readable language value. + options: + - en + - zh + - multilingual + validations: + required: true + - type: textarea + id: rationale + attributes: + label: Why it belongs + description: Explain the reader outcome, why this source is authoritative, and where it fits. Use your own words. + placeholder: This official guide helps intermediate readers... + validations: + required: true + - type: textarea + id: evidence + attributes: + label: Maintenance, licensing, and safety evidence + description: Include the review date, ownership or license, current maintenance evidence, API-key or paid-access requirements, and material risks. + validations: + required: true + - type: checkboxes + id: confirmations + attributes: + label: Submission checks + options: + - label: I read the curation policy and verified the canonical URL. + required: true + - label: The rationale and classification were reviewed by a person and were not generated by an LLM or web-search API. + required: true + - label: I disclosed any affiliation, sponsorship, or self-promotion in the additional context below. + required: true + - type: textarea + id: context + attributes: + label: Additional context or affiliation + description: State whether you maintain, work for, sponsor, or receive compensation from the resource. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..b82311f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,43 @@ +## Summary + +Describe the problem, the change, and the reader or maintainer outcome. + +## Change type + +- [ ] Catalog resource or classification +- [ ] Bilingual editorial content +- [ ] Site or build code +- [ ] Tests, tooling, or workflow +- [ ] Governance or documentation + +## Evidence and scope + +- Related issue: +- Source or maintenance evidence: +- User-visible pages affected: +- Intentionally out of scope: + +For catalog changes, explain the official source, review date, license or +ownership, maintenance state, API-key or payment requirements, and safety risk. +Disclose any affiliation or self-promotion. + +## Validation + +List the actual results, not only the commands you intended to run. + +- [ ] `python -m pytest` +- [ ] `python tools/validate_catalog.py` +- [ ] `bundle exec jekyll build` +- [ ] `bundle exec ruby tests/render_security_test.rb` +- [ ] English and Chinese output remain consistent +- [ ] No secrets, generated reports, local build output, or analytics IDs were committed + +Catalog links are fetched only by the scheduled or manually dispatched +workflow, not by pull-request CI. + +## Permissions and review + +- [ ] I have the right to submit this contribution. +- [ ] I retained third-party notices and did not relicense third-party material. +- [ ] Human reviewers wrote or verified all resource descriptions and classifications. +- [ ] I read `CONTRIBUTING.md` and `docs/CURATION_POLICY.md`. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..94a2b78 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,41 @@ +version: 2 +updates: + - package-ecosystem: bundler + directory: "/" + schedule: + interval: weekly + day: monday + time: "09:00" + timezone: Asia/Shanghai + open-pull-requests-limit: 5 + groups: + jekyll-dependencies: + patterns: + - "github-pages" + - "jekyll-*" + + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + day: monday + time: "09:15" + timezone: Asia/Shanghai + open-pull-requests-limit: 5 + groups: + python-dependencies: + patterns: + - "*" + + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + day: monday + time: "09:30" + timezone: Asia/Shanghai + open-pull-requests-limit: 5 + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/link-audit.yml b/.github/workflows/link-audit.yml new file mode 100644 index 0000000..8fc6814 --- /dev/null +++ b/.github/workflows/link-audit.yml @@ -0,0 +1,45 @@ +name: Catalog link audit + +on: + schedule: + - cron: "23 1 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: catalog-link-audit + cancel-in-progress: false + +jobs: + audit: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: ".python-version" + cache: pip + + - name: Install Python dependencies + run: python -m pip install --disable-pip-version-check -r requirements.lock.txt + + - name: Prepare report directory + run: mkdir -p reports + + - name: Check every catalog link + run: python tools/check_links.py --mode all --output reports/link-check.json + + - name: Upload link report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: catalog-link-report-${{ github.run_id }} + path: reports/link-check.json + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..2fc4042 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,48 @@ +name: Validate + +on: + pull_request: + push: + branches: + - master + +permissions: + contents: read + +concurrency: + group: validate-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: ".python-version" + cache: pip + + - name: Install Python dependencies + run: python -m pip install --disable-pip-version-check -r requirements-dev.lock.txt + + - name: Run tests + run: python -m pytest + + - name: Validate catalog + run: python tools/validate_catalog.py + + - name: Set up Ruby and Bundler + uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + bundler-cache: true + + - name: Build Jekyll site + run: bundle exec jekyll build + + - name: Verify catalog output escaping + run: bundle exec ruby tests/render_security_test.rb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..90520c2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ +# Jekyll +_site/ +.sass-cache/ +.jekyll-cache/ +.jekyll-metadata + +# Ruby +.bundle/ +vendor/ + +# OS +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Editor +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Logs +*.log + +# Runtime data +pids +*.pid +*.seed + +# Environment variables +.env +.env.local +.env.development.local +.env.test.local +.env.production.local + +# Temporary files +tmp/ +temp/ +.tmp/ + +# Python +.venv/ +venv/ +__pycache__/ +*.pyc + +# Generated reports +reports/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..28d9a01 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12.13 diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..a0891f5 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.3.4 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..3aef345 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,38 @@ +# FlyPython repository guide + +## Mission + +This repository is the broad, community-maintained resource catalog behind +FlyPython. Keep `flypython.com` focused on editorial learning paths and a small +featured set; keep this repository focused on the larger reviewed catalog and +its contribution workflow. + +## Content rules + +- Prefer official documentation and primary sources. +- Every catalog entry must include a stable ID, path, source type, level, + rationale, review date, status, and any key or safety requirements. +- Do not call a project production-ready without current maintenance, licensing, + security, and adoption evidence. +- Do not use an LLM or web-search API to generate resource descriptions. + Descriptions and classifications require human review. +- Keep English and Chinese pages generated from the same canonical resource + data. Do not hand-copy the full catalog into multiple Markdown files. +- Treat HTTP 403, 429, and transient 5xx responses as review-needed states, not + automatic proof that a resource is broken. + +## Change workflow + +- Work on a feature branch and preserve unrelated contributor changes. +- Run the repository validation workflow before committing. +- Keep external-link checks read-only, rate-limited, retryable, and blocked from + private or loopback network targets. +- Do not add API keys, tokens, analytics IDs, or generated reports to git. +- After merge, verify the GitHub Pages deployment and the production domain. + +## Review priorities + +1. Content accuracy and source quality. +2. Deterministic generation and bilingual parity. +3. Build, schema, link, accessibility, and SEO checks. +4. Clear contribution and licensing boundaries. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a58b41e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,39 @@ +# Changelog + +This file records notable repository and catalog-process changes. The Git +history remains the source for changes made before this changelog was added. + +## [Unreleased] + +### Added + +- A single bilingual catalog source with 21 reviewed primary resources across + Python foundations, web and APIs, automation, and AI-agent paths. +- Catalog schema validation and a safe external-link auditor with tests, + retry/backoff behavior, JSON reports, and SSRF/DNS-rebinding protection. +- English and Chinese catalog pages generated from the same resource data, + plus a 1200x630 social preview image. +- Contribution, conduct, security, and resource-curation policies. +- Structured issue forms and a pull request review checklist. +- Dependency update configuration and validation workflows. +- Scheduled, manually dispatchable catalog-link auditing that does not run on + untrusted pull requests. + +### Changed + +- Reframed the repository as the broad, community-maintained catalog behind + FlyPython while keeping the README files as short entry points. +- Pinned the Python 3.12 and Ruby 3.3 build toolchains and their validation + dependencies for reproducible local and CI checks. + +### Fixed + +- Corrected repository links, dynamic resource counts, bilingual metadata, + canonical and hreflang output, robots rules, and duplicate heading/meta tags. +- Escaped catalog-controlled HTML output, audited same-origin catalog links, + and made review-needed network results fail the scheduled audit. + +### Removed + +- Removed the legacy `README_cn` site route without adding a redirect, along + with unused feed, navigation, collection, theme, and inline-style config. diff --git a/CNAME b/CNAME new file mode 100644 index 0000000..bfb6cb6 --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +python.flypython.com \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3e295ed --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,124 @@ +# Contributor Covenant Code of Conduct + +## Our pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic +status, nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our standards + +Examples of behavior that contributes to a positive environment include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances + of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement responsibilities + +Community leaders are responsible for clarifying and enforcing our standards +of acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +hello@flypython.com. All complaints will be reviewed and investigated promptly +and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary ban + +**Community impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent ban + +**Community impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][mozilla-coc]. + +[homepage]: https://www.contributor-covenant.org +[mozilla-coc]: https://github.com/mozilla/diversity diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0883f29 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,102 @@ +# Contributing to FlyPython + +FlyPython is a bilingual, reviewed catalog of durable Python and AI-agent +resources. Contributions should improve accuracy, source quality, or the +catalog workflow. General Python questions belong in +[GitHub Discussions](https://github.com/flypythoncom/python/discussions). + +Before contributing, read the [curation policy](docs/CURATION_POLICY.md). + +## Permissions + +This repository currently does not grant a general license to reuse its +content or code. Public visibility is not permission to copy, redistribute, +or relicense repository material. Contributors must submit only material +they have the right to submit and must preserve any required third-party +notices. A contribution does not change the terms of third-party material. + +## Propose a change + +Use the matching issue form before a larger change: + +- **Resource proposal** for a new catalog entry. +- **Broken link** for an unreachable or replaced resource. +- **Security report** for a vulnerability. Follow [SECURITY.md](SECURITY.md) + instead of opening a public issue. + +Small typo, metadata, or maintenance fixes may go directly to a pull request. +Keep each pull request focused on one purpose. + +## Catalog changes + +`_data/resources.yml` is the canonical catalog source. Do not add or change a +resource only in a rendered Markdown page. + +Each resource entry must include these fields: + +- `id`, `path`, `title`, and `url` +- `source_type`, `level`, and `language` +- `why_en` and `why_zh` +- `reviewed_on` and `status` +- `requires_key`, `risk`, and `featured` + +Descriptions and classifications require human review. Do not use an LLM or a +web-search API to generate them. Prefer official documentation, official +standards, and official project pages. + +A 403, 429, or transient 5xx response is not enough to delete a resource. Mark +it for review and provide repeatable evidence. + +## Local setup + +Install the exact Python version from `.python-version`, Ruby, Bundler, and the +repository dependencies: + +```bash +python -m venv .venv +. .venv/bin/activate +python -m pip install -r requirements-dev.lock.txt +bundle install +``` + +## Required validation + +Run the same checks as CI before requesting review: + +```bash +python -m pytest +python tools/validate_catalog.py +bundle exec jekyll build +bundle exec ruby tests/render_security_test.rb +``` + +Maintainers can run the networked link audit from GitHub Actions with the +scheduled **Catalog link audit** workflow. For a deliberate local audit of +every catalog entry, including same-origin URLs: + +```bash +python tools/check_links.py --mode all --output reports/link-check.json +``` + +Network link fetching is intentionally excluded from pull-request CI. A pull +request can contain untrusted URLs, and status checks must remain deterministic. +Any review-needed link result fails the scheduled audit so a maintainer must +inspect it; it does not by itself justify removing the resource. + +## Pull request checklist + +- Keep catalog data and bilingual output consistent. +- Include evidence for maintenance, license, safety, and adoption claims. +- Do not call a project production-ready without current evidence. +- Do not commit secrets, API keys, analytics IDs, generated reports, or local + build output. +- Submit only material you have the right to submit and retain any required + third-party notice. +- Explain user-visible changes and list the commands you ran. + +## Review and merge + +Maintainers may request edits, reclassify an entry, or decline resources that +do not meet the curation policy. Passing automated checks does not replace +editorial review. After merge, maintainers verify the GitHub Pages deployment +and the production domain. diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..ede54ad --- /dev/null +++ b/Gemfile @@ -0,0 +1,5 @@ +source "https://rubygems.org" + +ruby "3.3.4" + +gem "github-pages", "= 232", group: :jekyll_plugins diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..3f27afd --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,329 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (8.1.3.1) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + json + logger (>= 1.4.2) + minitest (>= 5.1) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) + addressable (2.9.0) + public_suffix (>= 2.0.2, < 8.0) + base64 (0.3.0) + bigdecimal (4.1.2) + coffee-script (2.4.1) + coffee-script-source + execjs + coffee-script-source (1.12.2) + colorator (1.1.0) + commonmarker (0.23.12) + concurrent-ruby (1.3.8) + connection_pool (3.0.2) + csv (3.3.6) + dnsruby (1.74.0) + base64 (>= 0.2) + logger (~> 1.6) + simpleidn (~> 0.2.1) + drb (2.2.3) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + ethon (0.18.0) + ffi (>= 1.15.0) + logger + eventmachine (1.2.7) + execjs (2.10.2) + json (>= 2) + faraday (2.14.3) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.4) + net-http (~> 0.5) + ffi (1.17.4) + ffi (1.17.4-aarch64-linux-gnu) + ffi (1.17.4-aarch64-linux-musl) + ffi (1.17.4-arm-linux-gnu) + ffi (1.17.4-arm-linux-musl) + ffi (1.17.4-arm64-darwin) + ffi (1.17.4-x86-linux-gnu) + ffi (1.17.4-x86-linux-musl) + ffi (1.17.4-x86_64-darwin) + ffi (1.17.4-x86_64-linux-gnu) + ffi (1.17.4-x86_64-linux-musl) + forwardable-extended (2.6.0) + gemoji (4.1.0) + github-pages (232) + github-pages-health-check (= 1.18.2) + jekyll (= 3.10.0) + jekyll-avatar (= 0.8.0) + jekyll-coffeescript (= 1.2.2) + jekyll-commonmark-ghpages (= 0.5.1) + jekyll-default-layout (= 0.1.5) + jekyll-feed (= 0.17.0) + jekyll-gist (= 1.5.0) + jekyll-github-metadata (= 2.16.1) + jekyll-include-cache (= 0.2.1) + jekyll-mentions (= 1.6.0) + jekyll-optional-front-matter (= 0.3.2) + jekyll-paginate (= 1.1.0) + jekyll-readme-index (= 0.3.0) + jekyll-redirect-from (= 0.16.0) + jekyll-relative-links (= 0.6.1) + jekyll-remote-theme (= 0.4.3) + jekyll-sass-converter (= 1.5.2) + jekyll-seo-tag (= 2.8.0) + jekyll-sitemap (= 1.4.0) + jekyll-swiss (= 1.0.0) + jekyll-theme-architect (= 0.2.0) + jekyll-theme-cayman (= 0.2.0) + jekyll-theme-dinky (= 0.2.0) + jekyll-theme-hacker (= 0.2.0) + jekyll-theme-leap-day (= 0.2.0) + jekyll-theme-merlot (= 0.2.0) + jekyll-theme-midnight (= 0.2.0) + jekyll-theme-minimal (= 0.2.0) + jekyll-theme-modernist (= 0.2.0) + jekyll-theme-primer (= 0.6.0) + jekyll-theme-slate (= 0.2.0) + jekyll-theme-tactile (= 0.2.0) + jekyll-theme-time-machine (= 0.2.0) + jekyll-titles-from-headings (= 0.5.3) + jemoji (= 0.13.0) + kramdown (= 2.4.0) + kramdown-parser-gfm (= 1.1.0) + liquid (= 4.0.4) + mercenary (~> 0.3) + minima (= 2.5.1) + nokogiri (>= 1.16.2, < 2.0) + rouge (= 3.30.0) + terminal-table (~> 1.4) + webrick (~> 1.8) + github-pages-health-check (1.18.2) + addressable (~> 2.3) + dnsruby (~> 1.60) + octokit (>= 4, < 8) + public_suffix (>= 3.0, < 6.0) + typhoeus (~> 1.3) + html-pipeline (2.14.3) + activesupport (>= 2) + nokogiri (>= 1.4) + http_parser.rb (0.8.1) + i18n (1.15.2) + concurrent-ruby (~> 1.0) + jekyll (3.10.0) + addressable (~> 2.4) + colorator (~> 1.0) + csv (~> 3.0) + em-websocket (~> 0.5) + i18n (>= 0.7, < 2) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (>= 1.17, < 3) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + webrick (>= 1.0) + jekyll-avatar (0.8.0) + jekyll (>= 3.0, < 5.0) + jekyll-coffeescript (1.2.2) + coffee-script (~> 2.2) + coffee-script-source (~> 1.12) + jekyll-commonmark (1.4.0) + commonmarker (~> 0.22) + jekyll-commonmark-ghpages (0.5.1) + commonmarker (>= 0.23.7, < 1.1.0) + jekyll (>= 3.9, < 4.0) + jekyll-commonmark (~> 1.4.0) + rouge (>= 2.0, < 5.0) + jekyll-default-layout (0.1.5) + jekyll (>= 3.0, < 5.0) + jekyll-feed (0.17.0) + jekyll (>= 3.7, < 5.0) + jekyll-gist (1.5.0) + octokit (~> 4.2) + jekyll-github-metadata (2.16.1) + jekyll (>= 3.4, < 5.0) + octokit (>= 4, < 7, != 4.4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-mentions (1.6.0) + html-pipeline (~> 2.3) + jekyll (>= 3.7, < 5.0) + jekyll-optional-front-matter (0.3.2) + jekyll (>= 3.0, < 5.0) + jekyll-paginate (1.1.0) + jekyll-readme-index (0.3.0) + jekyll (>= 3.0, < 5.0) + jekyll-redirect-from (0.16.0) + jekyll (>= 3.3, < 5.0) + jekyll-relative-links (0.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-remote-theme (0.4.3) + addressable (~> 2.0) + jekyll (>= 3.5, < 5.0) + jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) + rubyzip (>= 1.3.0, < 3.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) + jekyll-swiss (1.0.0) + jekyll-theme-architect (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-cayman (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-hacker (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-leap-day (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-merlot (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-midnight (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-minimal (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-modernist (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-primer (0.6.0) + jekyll (> 3.5, < 5.0) + jekyll-github-metadata (~> 2.9) + jekyll-seo-tag (~> 2.0) + jekyll-theme-slate (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.3) + jekyll (>= 3.3, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + jemoji (0.13.0) + gemoji (>= 3, < 5) + html-pipeline (~> 2.2) + jekyll (>= 3.0, < 5.0) + json (2.21.2) + kramdown (2.4.0) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.10.0) + logger + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + logger (1.7.0) + mercenary (0.3.6) + mini_portile2 (2.8.9) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (6.0.6) + drb (~> 2.0) + prism (~> 1.5) + net-http (0.9.1) + uri (>= 0.11.1) + nokogiri (1.19.4) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.19.4-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.4-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.19.4-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.4-arm-linux-musl) + racc (~> 1.4) + nokogiri (1.19.4-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.4-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.4-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.4-x86_64-linux-musl) + racc (~> 1.4) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + prism (1.9.0) + public_suffix (5.1.1) + racc (1.8.1) + rb-fsevent (0.11.2) + rb-inotify (0.11.1) + ffi (~> 1.0) + rexml (3.4.4) + rouge (3.30.0) + rubyzip (2.4.1) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + sawyer (0.9.3) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) + securerandom (0.4.1) + simpleidn (0.2.3) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) + typhoeus (1.6.0) + ethon (>= 0.18.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (1.8.0) + uri (1.1.1) + webrick (1.9.2) + +PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + arm64-darwin-25 + ruby + x86-linux-gnu + x86-linux-musl + x86_64-darwin + x86_64-linux + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + github-pages (= 232) + +RUBY VERSION + ruby 3.3.4p94 + +BUNDLED WITH + 2.5.11 diff --git a/README.md b/README.md index 94b5764..116584f 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,49 @@ -python -====== -学习Python时的代码 +# FlyPython Python Resource Catalog +[![GitHub stars](https://img.shields.io/github/stars/flypythoncom/python?style=flat-square&label=stars)](https://github.com/flypythoncom/python/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/flypythoncom/python?style=flat-square&label=forks)](https://github.com/flypythoncom/python/forks) +[![Catalog](https://img.shields.io/badge/catalog-reviewed-157878?style=flat-square)](https://python.flypython.com/) +[English](README.md) · [中文](README_cn.md) +FlyPython is the broad, community-maintained resource catalog behind the +[FlyPython learning hub](https://flypython.com/). It keeps the larger source map +public and reviewable while the main site provides editorial learning paths and a +smaller featured collection. +## Start here -####2016-03-20 update +- [Follow the practical Python roadmap](https://flypython.com/learn) +- [Browse the featured resource index](https://flypython.com/resources) +- [Build and test a no-key Python agent loop](https://flypython.com/learn/python-ai-agent-roadmap) +- [Explore the full community catalog](https://python.flypython.com/) -- [Machine Learning with Python](https://github.com/xxg1413/MachineLearning) -- [Python-Algorithms](https://github.com/xxg1413/Python-Algorithms) -- [python-cookbook](https://github.com/xxg1413/python-cookbook) +## Four paths + +1. **Python foundations** — language, environments, dependencies, typing, and tests +2. **Web and APIs** — typed services, validation, HTTP clients, and applications +3. **Automation** — files, processes, browsers, crawling, and data workflows +4. **AI agents** — tools, structured output, state, evaluation, and safety boundaries + +The canonical catalog lives in [`_data/resources.yml`](_data/resources.yml). +The English and Chinese site pages read from that same file; the full resource +list is not copied into the README files. + +## Quality bar + +- Prefer official documentation, standards, and first-party project sources. +- Explain why each resource matters instead of publishing an unranked link dump. +- Record the review date, API-key requirement, and relevant safety boundary. +- Label experiments honestly; do not infer production readiness from popularity. + +See [CONTRIBUTING.md](CONTRIBUTING.md) before proposing a resource or changing +its classification. + +## Local preview + +```bash +bundle install +bundle exec jekyll serve +``` + +The catalog is published at [python.flypython.com](https://python.flypython.com/). diff --git a/README_cn.md b/README_cn.md new file mode 100644 index 0000000..27a58dd --- /dev/null +++ b/README_cn.md @@ -0,0 +1,45 @@ +# FlyPython Python 资源目录 + +[![GitHub stars](https://img.shields.io/github/stars/flypythoncom/python?style=flat-square&label=stars)](https://github.com/flypythoncom/python/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/flypythoncom/python?style=flat-square&label=forks)](https://github.com/flypythoncom/python/forks) +[![Catalog](https://img.shields.io/badge/catalog-reviewed-157878?style=flat-square)](https://python.flypython.com/zh/) + +[English](README.md) · [中文](README_cn.md) + +FlyPython 是 [FlyPython 学习站](https://flypython.com/)背后的社区公开资源目录。 +本仓库维护覆盖面更广、可公开审核的来源地图；主站负责编辑型学习路径和更精简的精选资源。 + +## 从这里开始 + +- [查看实用 Python 学习路线](https://flypython.com/learn) +- [浏览主站精选资源](https://flypython.com/resources) +- [构建并测试无需 API Key 的 Python Agent 循环](https://flypython.com/learn/python-ai-agent-roadmap) +- [浏览完整中文目录](https://python.flypython.com/zh/) + +## 四条路径 + +1. **Python 基础** — 语言、环境、依赖、类型与测试 +2. **Web 与 API** — 类型化服务、数据验证、HTTP 客户端与应用 +3. **自动化** — 文件、进程、浏览器、爬取与数据工作流 +4. **AI Agent** — 工具、结构化输出、状态、评测与安全边界 + +唯一的目录数据源是 [`_data/resources.yml`](_data/resources.yml)。英文页和中文页 +都读取同一份数据，不再在多个 Markdown 文件中复制完整资源清单。 + +## 收录标准 + +- 优先选择官方文档、正式标准与项目的一手来源。 +- 说明每条资源为什么值得使用，而不是堆放未经筛选的链接。 +- 记录审核日期、API Key 要求与需要关注的安全边界。 +- 如实标记实验性内容，不用流行度推断“生产可用”。 + +提交新资源或修改分类前，请先阅读 [CONTRIBUTING.md](CONTRIBUTING.md)。 + +## 本地预览 + +```bash +bundle install +bundle exec jekyll serve +``` + +目录发布在 [python.flypython.com/zh/](https://python.flypython.com/zh/)。 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..7fd314e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,48 @@ +# Security policy + +## Supported versions + +Security fixes apply to the default branch and the currently deployed +`python.flypython.com` site. Historical commits, forks, and archived catalog +entries are not supported releases. + +## Report a vulnerability privately + +Do not open a public issue for a security vulnerability. Use +[GitHub's private vulnerability reporting](https://github.com/flypythoncom/python/security/advisories/new). +If that form is unavailable, email hello@flypython.com with the subject +`FlyPython security report`. + +Include: + +- The affected URL, file, workflow, or commit +- Clear reproduction steps and impact +- Any proof of concept needed to confirm the issue +- Whether the issue is already public +- A safe way to contact you + +Remove credentials and personal data that are not required to reproduce the +issue. We will acknowledge reports on a best-effort basis, investigate, and +coordinate disclosure after a fix is available. + +## In scope + +Examples include: + +- Script injection or unsafe rendered catalog content +- A link-checking path that can reach private, loopback, or metadata services +- Workflow permission escalation or untrusted-code execution +- Exposed credentials, DNS takeover, or custom-domain control issues +- Dependency or build-chain compromise with a demonstrated impact + +Broken links, outdated descriptions, ordinary 403/429 responses, and resource +quality disagreements are not security vulnerabilities. Use the broken-link or +resource-proposal issue form for those reports. + +## Safe harbor + +Make a good-faith effort to avoid privacy violations, service disruption, data +destruction, and access beyond what is needed to demonstrate the issue. Do not +perform denial-of-service testing or interact with third-party resources beyond +their published policies. We will not pursue action against good-faith research +that follows this policy. diff --git a/_config.yml b/_config.yml new file mode 100644 index 0000000..075862b --- /dev/null +++ b/_config.yml @@ -0,0 +1,73 @@ +title: FlyPython Resource Catalog +description: >- + A reviewed catalog of primary Python sources for foundations, web APIs, + automation, and AI agents. +url: "https://python.flypython.com" +baseurl: "" +repository: flypythoncom/python +timezone: Asia/Shanghai + +author: + name: FlyPython + url: https://flypython.com/ + email: hello@flypython.com + github: flypythoncom + +lang: en-US +logo: /assets/images/logo.svg + +social: + name: FlyPython + links: + - https://flypython.com/ + - https://github.com/flypythoncom + +markdown: kramdown +highlighter: rouge +permalink: pretty +plugins: + - jekyll-sitemap + - jekyll-seo-tag + +optional_front_matter: + enabled: false + +readme_index: + enabled: false + +relative_links: + enabled: false + +titles_from_headings: + enabled: false + +kramdown: + input: GFM + hard_wrap: false + syntax_highlighter: rouge + +exclude: + - .github/ + - .gitignore + - .python-version + - .ruby-version + - AGENTS.md + - CHANGELOG.md + - CODE_OF_CONDUCT.md + - CONTRIBUTING.md + - README.md + - README_cn.md + - SECURITY.md + - Gemfile + - Gemfile.lock + - docs/ + - pyproject.toml + - requirements-dev.lock.txt + - requirements-dev.txt + - requirements.lock.txt + - requirements.txt + - node_modules/ + - vendor/ + - tools/ + - tests/ + - reports/ diff --git a/_data/resources.yml b/_data/resources.yml new file mode 100644 index 0000000..4798af9 --- /dev/null +++ b/_data/resources.yml @@ -0,0 +1,344 @@ +catalog: + reviewed_on: "2026-08-31" + status: active + paths: + - id: foundations + order: 1 + title_en: "Python foundations" + title_zh: "Python 基础" + summary_en: "Learn the language, environments, dependencies, typing, and tests that reliable Python work depends on." + summary_zh: "掌握语言基础、环境、依赖、类型与测试，为可靠的 Python 项目打好根基。" + - id: web-apis + order: 2 + title_en: "Web and APIs" + title_zh: "Web 与 API" + summary_en: "Build typed services and applications that connect Python logic to users and other systems." + summary_zh: "构建有类型约束的服务和应用，把 Python 逻辑连接到用户与其他系统。" + - id: automation + order: 3 + title_en: "Automation" + title_zh: "自动化" + summary_en: "Turn repeatable work into maintainable scripts, browser workflows, and data pipelines." + summary_zh: "把重复工作变成可维护的脚本、浏览器工作流和数据管道。" + - id: ai-agents + order: 4 + title_en: "AI agents" + title_zh: "AI Agent" + summary_en: "Learn tools, structured output, state, evaluation, and the safety boundaries of agent systems." + summary_zh: "理解工具调用、结构化输出、状态、评测以及 Agent 系统的安全边界。" + +resources: + - id: python-tutorial + path: foundations + title: "The Python Tutorial" + url: "https://docs.python.org/3/tutorial/" + source_type: official-docs + level: beginner + language: en + why_en: "The primary introduction to Python syntax, data structures, modules, errors, classes, and the standard library." + why_zh: "Python 官方入门教程，覆盖语法、数据结构、模块、异常、类与标准库。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: true + + - id: python-venv + path: foundations + title: "venv — Creation of virtual environments" + url: "https://docs.python.org/3/library/venv.html" + source_type: official-docs + level: beginner + language: en + why_en: "The standard-library reference for creating isolated Python environments and understanding their behavior." + why_zh: "Python 标准库的虚拟环境参考，说明如何创建隔离环境及其运行方式。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: python-packaging-guide + path: foundations + title: "Python Packaging User Guide" + url: "https://packaging.python.org/en/latest/tutorials/packaging-projects/" + source_type: official-docs + level: intermediate + language: en + why_en: "The PyPA tutorial for project layout, build metadata, distributions, and publishing Python packages." + why_zh: "PyPA 官方教程，覆盖项目结构、构建元数据、分发包与 Python 包发布。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: uv-docs + path: foundations + title: "uv documentation" + url: "https://docs.astral.sh/uv/" + source_type: official-project + level: all-levels + language: en + why_en: "A modern workflow for Python versions, virtual environments, dependencies, lockfiles, and project commands." + why_zh: "现代 Python 工作流，统一管理版本、虚拟环境、依赖、锁文件和项目命令。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: true + + - id: pytest-docs + path: foundations + title: "pytest documentation" + url: "https://docs.pytest.org/en/stable/" + source_type: official-project + level: intermediate + language: en + why_en: "The official guide to test discovery, assertions, fixtures, parametrization, and maintainable test suites." + why_zh: "pytest 官方指南，覆盖测试发现、断言、fixture、参数化和可维护测试套件。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: typing-docs + path: foundations + title: "Python typing documentation" + url: "https://typing.python.org/en/latest/" + source_type: official-docs + level: intermediate + language: en + why_en: "The canonical home for Python's type-system specification, guides, and typing best practices." + why_zh: "Python 类型系统规范、指南与类型标注实践的权威入口。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: fastapi-docs + path: web-apis + title: "FastAPI documentation" + url: "https://fastapi.tiangolo.com/" + source_type: official-docs + level: intermediate + language: en + why_en: "A direct route from typed Python functions to tested APIs with validation and generated documentation." + why_zh: "从带类型的 Python 函数出发，构建具备验证、测试和自动文档的 API。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: true + + - id: django-tutorial + path: web-apis + title: "Django documentation: First steps" + url: "https://docs.djangoproject.com/en/stable/intro/tutorial01/" + source_type: official-docs + level: beginner + language: en + why_en: "The official path through Django projects, models, views, templates, forms, tests, and reusable apps." + why_zh: "Django 官方入门路径，覆盖项目、模型、视图、模板、表单、测试和可复用应用。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: flask-docs + path: web-apis + title: "Flask documentation" + url: "https://flask.palletsprojects.com/en/stable/" + source_type: official-docs + level: beginner + language: en + why_en: "The maintained reference and tutorial for Flask applications, requests, templates, testing, and deployment." + why_zh: "Flask 官方参考与教程，覆盖应用、请求、模板、测试和部署。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: pydantic-docs + path: web-apis + title: "Pydantic documentation" + url: "https://pydantic.dev/docs/validation/latest/get-started/" + source_type: official-docs + level: intermediate + language: en + why_en: "The primary guide to validating untrusted data and expressing typed contracts at Python system boundaries." + why_zh: "在 Python 系统边界验证不可信数据、定义类型契约的官方指南。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: httpx-docs + path: web-apis + title: "HTTPX documentation" + url: "https://www.python-httpx.org/" + source_type: official-project + level: intermediate + language: en + why_en: "The official reference for synchronous and asynchronous HTTP clients, timeouts, streaming, and transport control." + why_zh: "同步与异步 HTTP 客户端、超时、流式响应和传输控制的官方参考。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: pathlib-docs + path: automation + title: "pathlib — Object-oriented filesystem paths" + url: "https://docs.python.org/3/library/pathlib.html" + source_type: official-docs + level: beginner + language: en + why_en: "The standard-library reference for readable, cross-platform file and directory automation." + why_zh: "使用可读、跨平台的对象式路径完成文件与目录自动化的标准库参考。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: subprocess-docs + path: automation + title: "subprocess — Subprocess management" + url: "https://docs.python.org/3/library/subprocess.html" + source_type: official-docs + level: intermediate + language: en + why_en: "The standard-library contract for launching processes, capturing output, handling failures, and avoiding unsafe shell usage." + why_zh: "启动进程、捕获输出、处理失败并避免不安全 shell 用法的标准库契约。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: medium + featured: false + + - id: playwright-python + path: automation + title: "Playwright for Python" + url: "https://playwright.dev/python/docs/intro" + source_type: official-docs + level: intermediate + language: en + why_en: "The official Python guide to reliable browser automation, locators, assertions, traces, and isolated contexts." + why_zh: "可靠浏览器自动化、定位器、断言、追踪和隔离上下文的 Python 官方指南。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: medium + featured: false + + - id: scrapy-tutorial + path: automation + title: "Scrapy tutorial" + url: "https://docs.scrapy.org/en/latest/intro/tutorial.html" + source_type: official-docs + level: intermediate + language: en + why_en: "The official introduction to crawl structure, extraction, item pipelines, persistence, and responsible operation." + why_zh: "爬虫结构、数据提取、item pipeline、持久化与负责任运行的官方入门。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: medium + featured: false + + - id: pandas-getting-started + path: automation + title: "pandas getting started guides" + url: "https://pandas.pydata.org/docs/getting_started/index.html" + source_type: official-docs + level: beginner + language: en + why_en: "The primary entry point for tabular data loading, cleaning, transformation, analysis, and export." + why_zh: "表格数据读取、清洗、转换、分析和导出的官方入门入口。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: low + featured: false + + - id: openai-agents-sdk + path: ai-agents + title: "OpenAI Agents SDK" + url: "https://openai.github.io/openai-agents-python/" + source_type: official-docs + level: intermediate + language: en + why_en: "The official Python toolkit for agents, tools, handoffs, guardrails, sessions, tracing, and orchestration." + why_zh: "面向 Agent、工具、handoff、guardrail、session、追踪与编排的官方 Python 工具包。" + reviewed_on: "2026-08-31" + status: active + requires_key: true + risk: medium + featured: true + + - id: pydantic-ai + path: ai-agents + title: "Pydantic AI" + url: "https://pydantic.dev/docs/ai/overview/" + source_type: official-docs + level: intermediate + language: en + why_en: "Typed agent development built around validated inputs, structured output, tools, testing, and model portability." + why_zh: "以输入验证、结构化输出、工具、测试和模型可移植性为核心的类型安全 Agent 开发。" + reviewed_on: "2026-08-31" + status: active + requires_key: true + risk: medium + featured: true + + - id: model-context-protocol + path: ai-agents + title: "Model Context Protocol" + url: "https://modelcontextprotocol.io/docs/getting-started/intro" + source_type: official-standard + level: intermediate + language: en + why_en: "The open protocol and reference documentation for connecting AI applications to tools, data, and reusable context." + why_zh: "连接 AI 应用、工具、数据与可复用上下文的开放协议和权威参考。" + reviewed_on: "2026-08-31" + status: active + requires_key: false + risk: medium + featured: true + + - id: langgraph-python + path: ai-agents + title: "LangGraph overview" + url: "https://docs.langchain.com/oss/python/langgraph/overview" + source_type: official-docs + level: advanced + language: en + why_en: "The official Python guide to stateful graph workflows, durable execution, memory, interrupts, and human review." + why_zh: "有状态图工作流、持久执行、记忆、中断与人工审核的 Python 官方指南。" + reviewed_on: "2026-08-31" + status: active + requires_key: true + risk: medium + featured: false + + - id: google-adk + path: ai-agents + title: "Google Agent Development Kit" + url: "https://adk.dev/" + source_type: official-docs + level: advanced + language: en + why_en: "Google's official documentation for developing, evaluating, and deploying modular agent systems." + why_zh: "Google 官方模块化 Agent 系统开发、评测与部署文档。" + reviewed_on: "2026-08-31" + status: active + requires_key: true + risk: medium + featured: false diff --git a/_includes/catalog-grid.html b/_includes/catalog-grid.html new file mode 100644 index 0000000..001c694 --- /dev/null +++ b/_includes/catalog-grid.html @@ -0,0 +1,83 @@ +{% assign ordered_paths = site.data.resources.catalog.paths | sort: "order" %} + + +{% for path in ordered_paths %} + {% assign path_resources = include.resources | where: "path", path.id %} +
+
+
+ {% if page.lang == "zh-CN" %}路径{% else %}Path{% endif %} 0{{ path.order | escape }} +

{% if page.lang == "zh-CN" %}{{ path.title_zh | escape }}{% else %}{{ path.title_en | escape }}{% endif %}

+
+

{% if page.lang == "zh-CN" %}{{ path.summary_zh | escape }}{% else %}{{ path.summary_en | escape }}{% endif %}

+
+ +
+ {% for resource in path_resources %} +
+
+ {% if page.lang == "zh-CN" %} + {% case resource.source_type %} + {% when "official-docs" %}官方文档 + {% when "official-standard" %}正式标准 + {% when "official-project" %}官方项目 + {% endcase %} + {% else %} + {{ resource.source_type | replace: "-", " " | escape }} + {% endif %} + {% if resource.featured %}{% if page.lang == "zh-CN" %}精选{% else %}Featured{% endif %}{% endif %} +
+

{{ resource.title | escape }}

+

{% if page.lang == "zh-CN" %}{{ resource.why_zh | escape }}{% else %}{{ resource.why_en | escape }}{% endif %}

+
    + {% if page.lang == "zh-CN" %} + {% case resource.level %} + {% when "beginner" %}
  • 入门
    • + {% when "intermediate" %}
  • 进阶
    • + {% when "advanced" %}
  • 高级
    • + {% when "all-levels" %}
  • 所有阶段
    • + {% endcase %} + {% case resource.language %} + {% when "en" %}
  • 来源语言：英语
    • + {% when "zh" %}
  • 来源语言：中文
    • + {% when "multilingual" %}
  • 来源语言：多语言
    • + {% endcase %} +
  • 审核于 {{ resource.reviewed_on | escape }}
    • + {% else %} +
  • {{ resource.level | replace: "-", " " | escape }}
    • + {% case resource.language %} + {% when "en" %}
  • English
    • + {% when "zh" %}
  • Chinese
    • + {% when "multilingual" %}
  • Multilingual
    • + {% endcase %} +
  • Reviewed {{ resource.reviewed_on | escape }}
    • + {% endif %} +
+ {% if resource.requires_key or resource.risk == "medium" %} +

+ {% if resource.requires_key %}{% if page.lang == "zh-CN" %}典型用法需要 API Key{% else %}API key required for typical use{% endif %}{% endif %} + {% if resource.requires_key and resource.risk == "medium" %}{% endif %} + {% if resource.risk == "medium" %}{% if page.lang == "zh-CN" %}请检查权限与外部副作用{% else %}Review permissions and side effects{% endif %}{% endif %} +

+ {% endif %} +
+ {% endfor %} +
+
+{% endfor %} diff --git a/_includes/head-custom.html b/_includes/head-custom.html new file mode 100644 index 0000000..4958d72 --- /dev/null +++ b/_includes/head-custom.html @@ -0,0 +1,13 @@ + + + + + + + +{% if page.lang == 'zh-CN' %} + +{% else %} + +{% endif %} + diff --git a/_layouts/default.html b/_layouts/default.html new file mode 100644 index 0000000..14b52df --- /dev/null +++ b/_layouts/default.html @@ -0,0 +1,76 @@ + + + + + + {% seo %} + + {% include head-custom.html %} + + {% assign active_resources = site.data.resources.resources | where: "status", "active" %} + + {% if page.lang == 'zh-CN' %} + + {% else %} + + {% endif %} + + + +
+ {{ content }} +
+ + + + diff --git a/assets/css/custom.css b/assets/css/custom.css new file mode 100644 index 0000000..8e576c6 --- /dev/null +++ b/assets/css/custom.css @@ -0,0 +1,540 @@ +:root { + --ink: #172a35; + --muted: #5b6d77; + --line: #d8e3e4; + --surface: #ffffff; + --surface-soft: #f2f8f8; + --brand: #0f6f70; + --brand-dark: #0a4f50; + --brand-soft: #dff1f0; + --accent: #f0b429; + --caution: #7b4c00; + --caution-bg: #fff6df; + --shadow: 0 18px 50px rgba(21, 55, 65, 0.08); + --content-width: 72rem; +} +* { + box-sizing: border-box; +} + +html { + scroll-behavior: smooth; +} + +@media (prefers-reduced-motion: reduce) { + html { + scroll-behavior: auto; + } +} + +body { + margin: 0; + color: var(--ink); + background: var(--surface); + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + line-height: 1.65; + text-rendering: optimizeLegibility; +} + +a { + color: var(--brand); + text-underline-offset: 0.18em; +} + +a:hover { + color: var(--brand-dark); +} + +a:focus-visible, +button:focus-visible { + outline: 3px solid var(--accent); + outline-offset: 3px; +} + +.skip-link { + position: fixed; + top: 0.75rem; + left: 0.75rem; + z-index: 100; + padding: 0.65rem 0.9rem; + color: #ffffff; + background: var(--brand-dark); + border-radius: 0.4rem; + transform: translateY(-180%); +} + +.skip-link:focus { + color: #ffffff; + transform: translateY(0); +} + +.site-header { + color: #ffffff; + background: + radial-gradient(circle at 86% 18%, rgba(255, 255, 255, 0.14), transparent 22rem), + linear-gradient(135deg, var(--brand) 0%, #183d4b 100%); +} + +.site-header-inner, +.main-content, +.site-footer-inner { + width: min(calc(100% - 2rem), var(--content-width)); + margin-inline: auto; +} + +.site-header-inner { + padding: 1.4rem 0 2rem; +} + +.brand-row, +.header-summary, +.site-footer-inner { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1.5rem; +} + +.site-brand { + display: inline-flex; + align-items: center; + gap: 0.75rem; + color: #ffffff; + font-size: 1.12rem; + font-weight: 750; + text-decoration: none; +} + +.site-brand:hover { + color: #ffffff; +} + +.site-brand small { + margin-left: 0.32rem; + color: rgba(255, 255, 255, 0.72); + font-size: 0.78rem; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.brand-mark { + display: grid; + width: 2.4rem; + height: 2.4rem; + place-items: center; + color: var(--brand-dark); + background: #ffffff; + border-radius: 0.7rem; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-weight: 800; +} + +.language-nav { + display: inline-flex; + padding: 0.22rem; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.22); + border-radius: 999px; +} + +.language-nav a { + display: inline-flex; + min-height: 2.75rem; + align-items: center; + padding: 0.35rem 0.7rem; + color: rgba(255, 255, 255, 0.84); + border-radius: 999px; + font-size: 0.84rem; + text-decoration: none; +} + +.language-nav a[aria-current="page"] { + color: var(--brand-dark); + background: #ffffff; + font-weight: 700; +} + +.header-summary { + align-items: flex-end; + margin-top: 2rem; +} + +.header-summary > p { + max-width: 39rem; + margin: 0; + color: rgba(255, 255, 255, 0.9); + font-size: clamp(1rem, 2vw, 1.18rem); +} + +.header-summary ul { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 0.35rem 1rem; + margin: 0; + padding: 0; + color: rgba(255, 255, 255, 0.78); + font-size: 0.8rem; + list-style: none; +} + +.header-summary a { + color: #ffffff; +} + +.main-content { + padding: clamp(2.5rem, 7vw, 5.25rem) 0; +} + +.main-content > h1 { + max-width: 50rem; + margin: 0 0 1rem; + color: var(--ink); + font-size: clamp(2.35rem, 7vw, 4.75rem); + line-height: 1.04; + letter-spacing: -0.045em; +} + +.main-content > h1 + p { + max-width: 48rem; + margin: 0; + color: var(--muted); + font-size: clamp(1.03rem, 2vw, 1.22rem); +} + +.main-content > h2 { + margin-top: 4.5rem; + font-size: clamp(1.7rem, 4vw, 2.35rem); + letter-spacing: -0.025em; +} + +.cta-row { + display: flex; + flex-wrap: wrap; + gap: 0.75rem; + margin: 2rem 0 1.4rem; +} + +.button { + display: inline-flex; + min-height: 2.75rem; + align-items: center; + justify-content: center; + padding: 0.65rem 1rem; + color: var(--brand-dark); + background: var(--surface); + border: 1px solid var(--line); + border-radius: 0.65rem; + font-weight: 700; + text-decoration: none; +} + +.button:hover { + color: var(--brand-dark); + background: var(--surface-soft); + border-color: #acc8c9; +} + +.button-primary, +.button-primary:hover { + color: #ffffff; + background: var(--brand); + border-color: var(--brand); +} + +.catalog-note { + display: flex; + flex-wrap: wrap; + gap: 0.25rem 0.55rem; + margin: 0; + color: var(--muted); + font-size: 0.88rem; +} + +.catalog-note strong { + color: var(--ink); +} + +.path-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 0.9rem; + margin: 3.5rem 0 5rem; +} + +.path-card { + display: flex; + min-height: 15.5rem; + flex-direction: column; + padding: 1.25rem; + color: var(--ink); + background: linear-gradient(180deg, #ffffff 0%, var(--surface-soft) 100%); + border: 1px solid var(--line); + border-radius: 1rem; + box-shadow: var(--shadow); + text-decoration: none; + transition: transform 160ms ease, border-color 160ms ease; +} + +.path-card:hover { + color: var(--ink); + border-color: #9dbfc0; + transform: translateY(-3px); +} + +.path-number, +.section-eyebrow { + color: var(--brand); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.76rem; + font-weight: 800; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.path-card strong { + margin: 1.6rem 0 0.55rem; + font-size: 1.15rem; + line-height: 1.3; +} + +.path-card > span:not(.path-number) { + color: var(--muted); + font-size: 1rem; +} + +.path-card small { + margin-top: auto; + padding-top: 1rem; + color: var(--brand); + font-weight: 700; +} + +.path-section { + margin: 5rem 0; + scroll-margin-top: 1.5rem; +} + +.section-heading { + display: grid; + grid-template-columns: minmax(14rem, 0.75fr) minmax(18rem, 1.25fr); + gap: 2rem; + align-items: end; + margin-bottom: 1.5rem; + padding-bottom: 1.2rem; + border-bottom: 1px solid var(--line); +} + +.section-heading h2 { + margin: 0.3rem 0 0; + font-size: clamp(1.75rem, 4vw, 2.5rem); + letter-spacing: -0.03em; +} + +.section-heading > p { + margin: 0; + color: var(--muted); +} + +.resource-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 1rem; +} + +.resource-card { + display: flex; + min-width: 0; + flex-direction: column; + padding: 1.25rem; + background: var(--surface); + border: 1px solid var(--line); + border-radius: 0.9rem; +} + +.resource-card.featured { + border-color: #86b8b9; + box-shadow: inset 0 3px 0 var(--brand); +} + +.resource-card-header { + display: flex; + justify-content: space-between; + gap: 0.75rem; +} + +.source-badge, +.featured-badge { + display: inline-flex; + align-items: center; + padding: 0.16rem 0.48rem; + border-radius: 999px; + font-size: 0.7rem; + font-weight: 800; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.source-badge { + color: var(--brand-dark); + background: var(--brand-soft); +} + +.featured-badge { + color: #6d4600; + background: #fff0bd; +} + +.resource-card h3 { + margin: 1rem 0 0.5rem; + font-size: 1.15rem; + line-height: 1.35; +} + +.resource-card h3 a { + color: var(--ink); + text-decoration-thickness: 1px; + text-decoration-color: #a9c8c9; +} + +.resource-card h3 a::after { + content: " ↗"; + color: var(--brand); + font-size: 0.78em; +} + +.resource-card > p:not(.resource-caution) { + margin: 0; + color: var(--muted); + font-size: 1rem; +} + +.resource-meta { + display: flex; + flex-wrap: wrap; + gap: 0.3rem 0.8rem; + margin: auto 0 0; + padding: 1.2rem 0 0; + color: var(--muted); + font-size: 0.74rem; + list-style: none; + text-transform: capitalize; +} + +.resource-meta li:not(:first-child)::before { + content: "·"; + margin-right: 0.8rem; + color: #9aa9ae; +} + +.resource-caution { + display: flex; + flex-wrap: wrap; + gap: 0.25rem 0.45rem; + margin: 0.85rem 0 0; + padding: 0.55rem 0.7rem; + color: var(--caution); + background: var(--caution-bg); + border-radius: 0.5rem; + font-size: 1rem; +} + +.site-footer { + padding: 2rem 0; + color: var(--muted); + background: var(--surface-soft); + border-top: 1px solid var(--line); +} + +.site-footer p { + margin: 0; +} + +.site-footer nav { + display: flex; + flex-wrap: wrap; + gap: 1rem; +} + +.site-footer a { + font-weight: 650; +} + +@media (max-width: 58rem) { + .path-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .path-card { + min-height: 13rem; + } +} + +@media (max-width: 42rem) { + .brand-row, + .header-summary, + .site-footer-inner { + align-items: flex-start; + flex-direction: column; + } + + .site-brand small { + display: block; + margin: 0; + } + + .header-summary ul { + justify-content: flex-start; + } + + .path-grid, + .resource-grid, + .section-heading { + grid-template-columns: 1fr; + } + + .path-grid { + margin-bottom: 4rem; + } + + .path-card { + min-height: 0; + } + + .section-heading { + gap: 0.75rem; + } + + .cta-row .button { + width: 100%; + } +} + +@media (prefers-reduced-motion: reduce) { + html { + scroll-behavior: auto; + } + + .path-card { + transition: none; + } +} + +@media print { + .site-header, + .site-footer, + .cta-row, + .path-grid { + display: none; + } + + .main-content { + width: 100%; + padding: 0; + } + + .resource-card { + break-inside: avoid; + } +} diff --git a/assets/images/logo.svg b/assets/images/logo.svg new file mode 100644 index 0000000..b57e2a9 --- /dev/null +++ b/assets/images/logo.svg @@ -0,0 +1,18 @@ + + FlyPython logo + A teal badge with a python glyph and FlyPython wordmark. + + + + + + + + + + 🐍 + + + FlyPython + + diff --git a/assets/images/og-image.png b/assets/images/og-image.png new file mode 100644 index 0000000..1709d98 Binary files /dev/null and b/assets/images/og-image.png differ diff --git a/assets/images/og-image.svg b/assets/images/og-image.svg new file mode 100644 index 0000000..da12ac7 --- /dev/null +++ b/assets/images/og-image.svg @@ -0,0 +1,28 @@ + + FlyPython social card + FlyPython social preview card for the reviewed Python resource catalog. + + + + + + + + + + + FlyPython + + + Reviewed Python resources for the AI era + + + Foundations · Web & APIs · Automation · AI agents + + + 🐍 + + + python.flypython.com + + diff --git a/datastruct/binarysearch.py b/datastruct/binarysearch.py deleted file mode 100644 index fa6a1e2..0000000 --- a/datastruct/binarysearch.py +++ /dev/null @@ -1,25 +0,0 @@ -def BinarySearch(l,key): - low=0 - high=len(l)-1 - i=0 - while(low <= high): - i = i+1 - mid = low + ((high-low)>>1) - if(l[mid] < key): - low = mid + 1 - elif (l[mid] > key): - high = mid -1 - else: - print "use %d times" % i - return mid - return -1 - -if __name__ == "__main__": - l=[1,4,5,6,7,8,9,44,333,2233] - print l - print BinarySearch(l,4) - print BinarySearch(l,44) - print BinarySearch(l,8) - print BinarySearch(l,2233) - print BinarySearch(l,77) - diff --git a/datastruct/btree.py b/datastruct/btree.py deleted file mode 100644 index 69434e8..0000000 --- a/datastruct/btree.py +++ /dev/null @@ -1,62 +0,0 @@ -class BTree: - def __init__(self,value): - self.left=None - self.data=value - self.right=None - - def insertLeft(self,value): - self.left=BTree(value) - return self.left - - def insertRight(self,value): - self.right=BTree(value) - return self.right - - def show(self): - print self.data - -def preorder(node): - if node.data: - node.show() - if node.left: - preorder(node.left) - if node.right: - preorder(node.right) - -def inorder(node): - if node.data: - if node.left: - inorder(node.left) - node.show() - if node.right: - inorder(node.right) - -def postorder(node): - if node.data: - if node.left: - postorder(node.left) - if node.right: - postorder(node.right) - node.show() - -if __name__ == "__main__": - - Root=BTree("root") - A=Root.insertLeft("A") - C=A.insertLeft("C") - D=C.insertRight("D") - F=D.insertLeft("F") - G=D.insertRight("G") - B=Root.insertRight("B") - E=B.insertRight("E") - - print "pre-traversal" - preorder(Root) - - print "in-traversal" - inorder(Root) - - print "post-traversal" - postorder(Root) - - diff --git a/datastruct/graph.py b/datastruct/graph.py deleted file mode 100644 index 834670b..0000000 --- a/datastruct/graph.py +++ /dev/null @@ -1,35 +0,0 @@ -def searchGraph(graph,start,end): - results=[] - generatePath(graph,[start],end,results) - results.sort(lambda x,y:cmp(len(x),len(y))) - return results - -def generatePath(graph,path,end,results): - state=path[-1] - if state == end: - results.append(path) - else: - for arc in graph[state]: - if arc not in path: - generatePath(graph,path+[arc],end,results) - - -if __name__ == "__main__": - Graph={ - 'A':['B','C','D'], - 'B':['E'], - 'C':['D','F'], - 'D':['B','E','G'], - 'E':[], - 'F':['D','G'], - 'G':['E'] - } - r = searchGraph(Graph,'A','D') - print "A to D" - for i in r: - print i - - r=searchGraph(Graph,'A','E') - print "A to E" - for i in r: - print i diff --git a/datastruct/queue.py b/datastruct/queue.py deleted file mode 100644 index 64da3e4..0000000 --- a/datastruct/queue.py +++ /dev/null @@ -1,43 +0,0 @@ -class Queue: - def __init__(self,size=20): - self.queue=[] - self.size=size - self.end=-1 - - def setSize(self,size): - self.size=size - - def In(self,element): - if self.end < self.size -1: - self.queue.append(element) - self.end = self.end + 1 - else: - raise "QueueFull" - - def Out(self): - if self.end != -1: - element = self.queue[0] - self.queue=self.queue[1:] - self.end = self.end-1 - return element - else: - raise "QueueEmpty" - - def End(self): - return self.end - - def empty(self): - self.queue=[] - self.end=-1 - -if __name__ == "__main__": - - queue=Queue() - for i in range(10): - queue.In(i) - print queue.End() - - for i in range(10): - print queue.Out() - - diff --git a/datastruct/sort.py b/datastruct/sort.py deleted file mode 100644 index b190701..0000000 --- a/datastruct/sort.py +++ /dev/null @@ -1,78 +0,0 @@ -class BTree: - def __init__(self,value): - self.left=None - self.data=value - self.right=None - - def insertLeft(self,value): - self.left=BTree(value) - return self.left - - def insertRight(self,value): - self.right=BTree(value) - return self.right - - def show(self): - print self.data - -def inorder(node): - if node.data: - if node.left: - inorder(node.left) - node.show() - if node.right: - inorder(node.right) - - -def rinorder(node): - if node.data: - if node.right: - rinorder(node.right) - node.show() - if node.left: - rinorder(node.left) - -def insert(node,value): - if value > node.data: - if node.right: - insert(node.right,value) - else: - node.insertRight(value) - else: - if node.left: - insert(node.left,value) - else: - node.insertLeft(value) - - -if __name__ == "__main__": - - l=[88,11,2,33,22,4,55,33,221,34] - Root=BTree(l[0]) - node=Root - for i in range(1,len(l)): - insert(Root,l[i]) - - print "1---->10" - inorder(Root) - print "10--->1" - rinorder(Root) - - - - - - - - - - - - - - - - - - - diff --git a/datastruct/stack.py b/datastruct/stack.py deleted file mode 100644 index e5b65ce..0000000 --- a/datastruct/stack.py +++ /dev/null @@ -1,59 +0,0 @@ -class Stack: - def __init__(self,size=20): - self.stack= [] - self.size= size; - self.top= -1 - - def setSize(self,size): - self.size=size; - - def push(self,element): - if self.isFull(): - raise "StackOverflow" - else: - self.stack.append(element) - self.top = self.top + 1 - - def pop(self): - if self.isEmpty(): - raise "StackUnderflow" - else: - element=self.stack[-1] - self.top=self.top-1; - del self.stack[-1] - return element - - def Top(self): - return self.top - - def empty(self): - self.stack=[] - self.top=-1 - - def isEmpty(self): - if self.top == -1: - return True - else: - return False - - def isFull(self): - if self.top == self.size-1: - return True - else: - return False - -if __name__ == "__main__": - - stack=Stack() - - for i in range(10): - stack.push(i) - print stack.Top() - - for i in range(10): - print stack.pop() - - stack.empty() - print stack.Top() - - diff --git a/docs/CURATION_POLICY.md b/docs/CURATION_POLICY.md new file mode 100644 index 0000000..12f87ac --- /dev/null +++ b/docs/CURATION_POLICY.md @@ -0,0 +1,136 @@ +# FlyPython curation policy + +This policy defines what belongs in the FlyPython catalog, what evidence a +resource needs, and how maintainers review changes. It is the reference for +resource proposals and catalog pull requests. + +## Scope + +FlyPython catalogs durable resources for Python learning and Python-based AI +agent development. The catalog favors material that helps readers build skills +or use a maintained tool safely. It is not a general link directory, product +launch feed, or hosting location for unrelated applications. + +Current learning paths are: + +- `foundations` +- `web-apis` +- `automation` +- `ai-agents` + +## Source priority + +Review sources in this order: + +1. Official documentation +2. Official standards +3. Official project repositories or sites + +The current `source_type` values are `official-docs`, `official-standard`, and +`official-project`. A new source type requires a schema and policy change in +the same pull request. + +Secondary tutorials, affiliate pages, copied lists, thin SEO pages, and +unmaintained mirrors are excluded unless the policy is deliberately expanded. + +## Required catalog fields + +`_data/resources.yml` is the canonical source. It contains a `catalog` object +and a `resources` list. + +The `catalog` object records: + +- `reviewed_on` +- `status` +- `paths`, where every path has `id`, `title_en`, `title_zh`, `summary_en`, + `summary_zh`, and `order` + +Every resource records: + +- Identity: `id`, `path`, `title`, `url` +- Classification: `source_type`, `level`, `language` +- Editorial rationale: `why_en`, `why_zh` +- Review state: `reviewed_on`, `status` +- Access and safety: `requires_key`, `risk` +- Editorial selection: `featured` + +Allowed classifications are: + +| Field | Values | +| --- | --- | +| `path` | `foundations`, `web-apis`, `automation`, `ai-agents` | +| `source_type` | `official-docs`, `official-standard`, `official-project` | +| `level` | `beginner`, `intermediate`, `advanced`, `all-levels` | +| `language` | `en`, `zh`, `multilingual` | +| `status` | `active` | +| `risk` | `low`, `medium` | +| `requires_key`, `featured` | Boolean | + +IDs must be stable, lowercase, and unique. A rename needs an explicit migration +plan because external links or generated anchors may depend on the old ID. +Review dates use ISO `YYYY-MM-DD` format. + +## Acceptance criteria + +A resource is eligible when all of these are true: + +- Its URL is canonical and controlled by the official publisher or project. +- Its purpose fits one learning path. +- The English and Chinese rationales are factual, specific, and human-reviewed. +- Access requirements, API keys, paid tiers, and material safety risks are + represented accurately. +- Its license and ownership are clear enough for the claims the catalog makes. +- The maintainer can verify the resource on the stated review date. + +Do not describe a project as production-ready without current maintenance, +licensing, security, and adoption evidence. Popularity alone is not enough. + +## Editorial rules + +Write short, original rationales that explain why the resource is useful in its +assigned path. Do not copy marketing claims or large passages from the source. +Do not use an LLM or web-search API to generate descriptions or classifications. +A human reviewer remains responsible for each statement. + +English and Chinese views must come from the same catalog record. A translation +should preserve the meaning and limits of the source, not add new claims. + +Self-promotion must be disclosed. Maintainers assess it under the same criteria +as every other proposal. + +## Link verification + +Internal validation runs on every pull request. Network link fetching runs only +on the scheduled or manually dispatched GitHub Actions workflow. + +A 404 or 410 from the canonical resource is strong removal evidence. A 403, 429, +timeout, or transient 5xx is a review-needed result, not proof that the resource +is gone. Review-needed results fail the automated audit so they cannot remain +hidden behind a green run. DNS, connection, TLS, invalid-URL, and +redirect-protocol failures also fail the audit, but none of these results alone +is enough to remove a resource. Retry with rate limits and record the observation +date. Link checks must refuse private, loopback, link-local, multicast, +cloud-platform, and cloud metadata targets, including redirect destinations. + +## Review cadence and removal + +Run the catalog-link audit weekly. Perform an editorial review when a resource +changes ownership, becomes unmaintained, introduces a material safety concern, +or receives a substantiated report. + +Remove a resource when it is permanently unavailable, outside the catalog scope, +materially misleading, malicious, or no longer meets the acceptance criteria. +The pull request should preserve the reason and evidence in its description. +Do not silently replace a resource with an unrelated alternative under the same +stable ID. + +## Permissions and attribution + +This repository currently does not grant a general license to reuse its content +or code. Public visibility is not permission to copy, redistribute, or relicense +repository material. Contributors must submit only material they have the right +to submit and must preserve required notices. + +Third-party content keeps its original terms. A catalog entry may link to and +factually describe a third-party resource, but it must not copy or relicense that +resource. diff --git a/expand/1.py b/expand/1.py deleted file mode 100644 index 3239a25..0000000 --- a/expand/1.py +++ /dev/null @@ -1,54 +0,0 @@ -#a = '19' -#print "please input a int number in 1-100" -#x = raw_input() -#if a != x: -# print "sorry you are wrong!\ninput again:" -# x = raw_input() -#else -# print "you are right" -#raw_input() - -from random import randint - -def printNum(): - - sInput = raw_input(u"请输入你猜的数字(1-100)：") - - try: - nInput = int(sInput) - except (ValueError,TypeError),diag: - print str(diag) - - if(nInput < 1 or nInput > 100): - print u"你数字不在范围之内，请重新输入" - - return nInput - -def main(): - - nValue = randint(1,100) - nInput = printNum() - nTotal = 1 - - while(nValue!= nInput): - - if (nValue > nInput): - print u"你猜的数小了" - - elif (nValue < nInput): - print u"你猜的数大了" - - nTotal += 1 - nInput=printNum() - - print u"恭喜你猜对了" - print u"你直到猜对 共猜了%d次" % nTotal - - if nTotal < 10: - print u"恭喜，你的成绩超过了平均水平" - else: - print u"很遗憾，你成绩没达到平均水平" - - -if __name__ == "__main__": - main() diff --git a/expand/2.py b/expand/2.py deleted file mode 100644 index 8f1d16f..0000000 --- a/expand/2.py +++ /dev/null @@ -1,27 +0,0 @@ -#s = raw_input() -#i = 0 -#for i in s: -# print i -#raw_input() - -#扩展成在字符串中把特定字符替换成规定字符 -import re -def printReplace(srcStr,word): - for letter in srcStr: - if letter == word: - srcStr = srcStr.replace(letter,'!') #字符串不可更改 - return srcStr - -def printReplace_re(srcStr,word): - srcStr=re.sub(word,'!',srcStr) - return srcStr - -def main(): - srcStr = raw_input(u"请输入一个字符串：") - destStr=printReplace_re(srcStr,'i') - print destStr - - -if __name__ == "__main__": - main() - diff --git a/index.md b/index.md new file mode 100644 index 0000000..01f793f --- /dev/null +++ b/index.md @@ -0,0 +1,44 @@ +--- +layout: default +title: Python Resource Catalog for the AI Era +description: A reviewed catalog of primary Python sources for foundations, web APIs, automation, and AI agents. +lang: en-US +permalink: / +image: + path: /assets/images/og-image.png + width: 1200 + height: 630 + alt: FlyPython Python Resource Catalog +--- +{% assign catalog_resources = site.data.resources.resources | where: "status", "active" %} + +# Learn Python from sources worth trusting. + +FlyPython is a reviewed map of official documentation, standards, and +first-party project resources. Use the four paths to find the next reliable +source, then return to the main learning hub for guided projects and context. + +
+ Start with the roadmap + Browse the featured index + Contribute on GitHub +
+ +

+ {{ catalog_resources.size }} active resources + + Catalog reviewed {{ site.data.resources.catalog.reviewed_on | escape }} + + Primary sources first +

+ +{% include catalog-grid.html resources=catalog_resources %} + +## How this catalog relates to FlyPython + +This repository keeps the broader community resource map public and reviewable. +The main [FlyPython learning hub](https://flypython.com/) turns a smaller set of +those sources into practical learning paths, tested examples, and clear next +steps. Start with the +[no-key Python agent guide](https://flypython.com/learn/python-ai-agent-roadmap) +if you want a complete project before choosing an agent framework. diff --git a/primer/10/card.py b/primer/10/card.py deleted file mode 100644 index e6bac33..0000000 --- a/primer/10/card.py +++ /dev/null @@ -1,33 +0,0 @@ -def safe_float(obj): - try: - retval = float(obj) - except (ValueError,TypeError),diag: - retval = str(diag) - return retval - -def main(): - log = open("cardlog.txt","w") - try: - ccfile = open("carddata.txt","r") - except IOError,e: - log.write("no txns this \n") - log.close() - return - txns = ccfile.readlines() - ccfile.close() - total = 0.00 - log.write("account log:\n") - - for eachTxn in txns: - result = safe_float(eachTxn) - if isinstance(result,float): - total += result - log.write("data ....processed\n") - else: - log.write("ignored: %s" % result) - print "$%.2f (new balance)" % (total) - log.close() - -if __name__ == '__main__': - print "run" - main() diff --git a/primer/10/myexc.py b/primer/10/myexc.py deleted file mode 100644 index 0e3245f..0000000 --- a/primer/10/myexc.py +++ /dev/null @@ -1,104 +0,0 @@ - -import os,socket,errno,types,tempfile - -class NetworkError(IOError): - pass - -class FileError(IOError): - pass - -def updArgs(args,newarg=None): - if isinstance(args,IOError): - myargs =[] - myargs.extend([arg for arg in args]) - else: - myargs = list(args) - - if newarg: - myargs.append(newarg) - - return tuple(myargs) - -def fileAargs(file,mode,agrs): - if args[0] == errno.EACCES and \ - 'access' in dir(os): - perms = '' - perms = {'r':os.R_OK,'w':os.W_OK,'x':os.X_OK} - pkeys = permd.keys() - pkeys.sort() - pkeys.reverse() - - for eachPerm in 'rwx': - if os.access(file,permd[eachPerm]): - perms += eachPerm - else: - perms += '-' - - if isinstance(args,IOError): - myargs = [] - myargs.extend([arg for arg in args]) - else: - myargs = list(args) - - myargs[1] = "'%s' %s (perms:'%s')" % (mode,myargs[1],perms) - - myargs.append(args.filename) - - else: - myargs = args - return tuple(myargs) - -def myconnnect(sock,host,port): - try: - sock.connect((hosy,port)) - except socket.error,args: - myargs = updArgs(args) - if len(myargs) == 1: - myargs = (errno.ENXIO,myargs[0]) - - raise NetworkError,\ - updArgs(myargs,host +":" + str(port)) -def myopen(file,mode ="r"): - try: - fo = open(file,mode) - except IOError,args: - raise FileError,fileArgs(file,mode,args) - return fo - -def testfile(): - file = mkdtemp() - f = open(file,"w") - f.close() - - for eachTest in ((0,"r"),(0100,"r"), \ - (0400,"w"),(0500,"w")): - try: - os.chmod(file,eachTest[0]) - f=myopen(file,eachTest[1]) - - except FileError,args: - print "%s: %s" % \ - (args.__class__.__name__,args) - else: - print file,"open ok..perm ignored" - f.close() - os.chmod(file,0777) - os.unlink(file) -def testnet(): - s = socket.socket(socket.AF_INET.socket.SOCK_STREAM) - for eachHost in ("deli","www"): - try: - myconnect(s,"deli",8080) - except NetworkError,args: - print "%s:%s" %\ - (args.__class__.__name__,args) -if __name__ == "__main__": - testfile() - testnet() - - - - - - - diff --git a/primer/11/Gui.py b/primer/11/Gui.py deleted file mode 100644 index 6bff1f6..0000000 --- a/primer/11/Gui.py +++ /dev/null @@ -1,17 +0,0 @@ - -from functools import partial -import Tkinter - -root=Tkinter.Tk() -MyButton = partial(Tkinter.Button,root, - fg="white",bg="blue") -b1= MyButton(text="button 1") -b2=MyButton(text="button 2") -#qb=MyButton(text="QUIT",bg="red", - # command=root.quit) - -b1.pack() -b2.pack() -#qb.pack(file=Tkinter.X,expand= True) -root.title("PFAs!") -root.mainloop() diff --git a/primer/11/deco.py b/primer/11/deco.py deleted file mode 100644 index 21d7389..0000000 --- a/primer/11/deco.py +++ /dev/null @@ -1,22 +0,0 @@ - -from time import ctime,sleep - -def tsfunc(func): - def wrappedFunc(): - print "[%s] %s() called" % \ - (ctime(),func.__name__) - return func() - return wrappendFunc - -@tsfunc -def foo(): - pass - -foo() -sleep(4) - -for i in range(2): - sleep(1) - foo() - - diff --git a/primer/11/easyMath.py b/primer/11/easyMath.py deleted file mode 100644 index e14d53b..0000000 --- a/primer/11/easyMath.py +++ /dev/null @@ -1,40 +0,0 @@ - -from operator import add,sub -from random import randint,choice - -ops ={"+":add,"-":sub} -MAXTRIES = 2 - -def doprob(): - op = choice('+-') - nums = [randint(1,10) for i in range(2)] - nums.sort(reverse = True) - ans = ops[op](*nums) - pr="%d %s %d =" %(nums[0],op,nums[1]) - oops =0 - while True: - try: - if int(raw_input(pr)) == ans: - print "corrent" - break - if oops == MAXTRIES: - print "answer \n %s%d" %(pr,ans) - else: - print "incorrent...try again" - oops += 1 - except (KeyboardInterrupt, EOFError,ValueError): - print "invalid input ... try again" - -def main(): - while True: - doprob() - try: - opt= raw_input("Again?[y]".lower()) - if opt and opt[0] == "n": - break - except (KeyboardInterrupt,EOFError): - break -if __name__ == "__main__": - main() - - diff --git a/primer/11/grabWeb.py b/primer/11/grabWeb.py deleted file mode 100644 index 5b3de06..0000000 --- a/primer/11/grabWeb.py +++ /dev/null @@ -1,26 +0,0 @@ - -from urllib import urlretrieve - -def firstNoBlank(lines): - for eachLine in lines: - if not eachLine.strip(): - continue - else: - return eachLine - -def firstLast(webpage): - f = open(webpage) - lines= f.readlines() - f.close() - print firstNoBlank(lines),lines.reverse() - print firstNoBlank(lines), - -def download(url = "http://www.cqupt.edu.cn",process=firstLast): - try: - retval = urlretrieve(url)[0] - except IOError: - retval = None - if retval: - process(retval) -if __name__ == "__main__": - download() diff --git a/primer/11/numconv.py b/primer/11/numconv.py deleted file mode 100644 index 63eb282..0000000 --- a/primer/11/numconv.py +++ /dev/null @@ -1,8 +0,0 @@ - -def convert(func,seq): - return[func (eachNum ) for eachNum in seq] - -myseq=(123,45.67,-6.2e8,99999999L) -print convert(int,myseq) -print convert(long,myseq) -print convert(float,myseq) diff --git a/primer/11/odd.py b/primer/11/odd.py deleted file mode 100644 index 2a37d6a..0000000 --- a/primer/11/odd.py +++ /dev/null @@ -1,10 +0,0 @@ - -from random import randint - -#def odd(n): -# return n%2 - -allNums =[] -for eachNum in range(9): - allNums.append(randint(1,99)) -print[n for n in allNums if n%2] diff --git a/primer/11/testit.py b/primer/11/testit.py deleted file mode 100644 index ad5037e..0000000 --- a/primer/11/testit.py +++ /dev/null @@ -1,26 +0,0 @@ - -def testit(func,*nkwargs,**kwargs): - try: - retval = func(*nkwargs,**kwargs) - result = (True,retval) - except Exception,diag: - result = (False,str(diag)) - return result - -def test(): - funcs =(int,long,float) - vals = (1234,12.34,"1234","12.34") - - for eachFunc in funcs: - print "_"*20 - for eachVal in vals: - retval = testit(eachFunc,eachVal) - if retval[0]: - print "%s(%s)=" % \ - (eachFunc.__name__,"eachVal"),retval[1] - else: - print "%s(%s)=FAILED:" %\ - (eachFunc.__name__,"eachVal"),retval[1] - -if __name__ == "__main__": - test() diff --git a/primer/12/hot.py b/primer/12/hot.py deleted file mode 100644 index e695791..0000000 --- a/primer/12/hot.py +++ /dev/null @@ -1,10 +0,0 @@ -class HotelRoomCalc(object): - 'hotel room rate calculator' - def __init__(self,rt,sales=0.085,rm=0.1): - self.salesTax = sales - self.roomTax = rm - self.roomRate = rt - - def calcTotal(self,days = 1): - daily = round((self.roomRate *(1+self.roomTax + self.salesTax)),2) - return float(days)*daily diff --git a/primer/13/NumStr.py b/primer/13/NumStr.py deleted file mode 100644 index 986062b..0000000 --- a/primer/13/NumStr.py +++ /dev/null @@ -1,27 +0,0 @@ -class NumStr(object): - def __init__(self,num=0,string=''): - self.__num = num - self.__string = string - - def __str__(self): - return "[%d :: %r]" % (self.__num,self.__string) - __repr__ = __str__ - - def __add__(self,other): - if isinstance(other,NumStr): - return self.__class__(self.__num + \ - other.__num,self.__string+other.__string) - else: - return TypeError,"type error" - def __mul__(self,num): - if isinstance(num,int): - return self.__class__(self.__num *num,self.__string *num) - else: - raise TypeError,"__num__ error" - def __nonzero__(self): - return self.__num or len(self.__string) - def __norm_cval(self,cmpres): - return cmp(cmpres,0) - def __cmp__(self,other): - return self.__norm_cval(cmp(self.__num,other.__num)) + \ - self.__norm_cval(cmp(self.__string,other.__string)) diff --git a/primer/13/Rand.py b/primer/13/Rand.py deleted file mode 100644 index 1b56d0d..0000000 --- a/primer/13/Rand.py +++ /dev/null @@ -1,10 +0,0 @@ - -from random import choice - -class Rand(object): - def _init_(self,seq): - self.data = seq - def _iter_(self): - return self - def next(self): - return choice(self.data) diff --git a/primer/13/file.py b/primer/13/file.py deleted file mode 100644 index 8830909..0000000 --- a/primer/13/file.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import pickle - -class File(object): - saved=[] - def __init__(self,name=None): - self.name = name - def __get__(self,obj,typ=None): - if self.name not in File.saved: - raise AttributeError,"%r used before assignment " % self.name - try: - f = open(self.name,"r") - val = pickle.load(f) - f.close() - return val - except (pickle.UnpicklingError,IOError,EOFError,AttributeError,\ - ImportError,IndexError),e: - raise AttributeError,"could not read %r:%s" % (self.name,e) - def __set__(self,obj,val): - f = open(self.name,"w") - try: - pickle.dump(val,f) - File.saved.append(self.name) - except (TypeError,pickle.PicklingError),e: - raise AttributeError,"could not pickle %r " % self.name - finally: - f.close() - - def __delete__(self,obj): - try: - os.unlink(self.name) - File.saved.remove(self.name) - except (OSError,ValueError),e: - pass diff --git a/primer/13/meta.py b/primer/13/meta.py deleted file mode 100644 index 243a253..0000000 --- a/primer/13/meta.py +++ /dev/null @@ -1,28 +0,0 @@ -from warnings import warn - -class ReqStr(type): - def __init__(cls,name,bases,attrd): - super(ReqStr,cls).__init__(name,bases,attrd) - if "__str__" not in attrd: - raise TypeError("class overring __str__") - if "__repr__" not in attrd: - warn("class suggests __repr__",stacklevel = 3) -print "define ReqStr (meta)class \n" - -class Foo(object): - __metaclass__ = ReqStr - - def __str__(self): - return "instance of class",self.__class__.__name__ -print "defined Foo class\n" - -class Bar(object): - __metaclass__ = ReqStr - - def __str__(self): - return self.__class__.__name__ -print "defined Bar class\n " - -class FooBar(object): - __metaclass__ = ReqStr -print "defined FooBar class \n" diff --git a/primer/13/roundFloat.py b/primer/13/roundFloat.py deleted file mode 100644 index 06cf9bf..0000000 --- a/primer/13/roundFloat.py +++ /dev/null @@ -1,9 +0,0 @@ -class RoundFloat(object): - def __init__(self,val): - assert isinstance(val,float),"Value must be a float" - self.value = round(val,2) - - def __str__(self): - return "%.2f" %self.value - __repr__ = __str__ - diff --git a/primer/13/time.py b/primer/13/time.py deleted file mode 100644 index de514b5..0000000 --- a/primer/13/time.py +++ /dev/null @@ -1,28 +0,0 @@ - -from time import time,ctime - -class Time(object): - def __init__(self,obj): - self.__data = obj; - self.__ctime = self.__mtime =self.__atime=time() - def get(self): - self.__atime = time() - return self.__data - def gettimeval(self,t_type): - if not isinstance(t_type,str) or t_type[0] not in "cma": - raise TypeError," arg c m a" - return getattr(self,"_%s__%stime" % (self.__class__.__name__,t_type[0])) - def gettimestr(self,t_type): - return ctime(self.gettimeval(t_type)) - def set(self,obj): - self.__data = obj; - self.__mtime = self.__atime= time() - def __repr__(self): - self.__atime = time() - return 'self.__data' - def __str__(self): - self.__atime = time() - return str(self.__data) - def __getatt__(self,attr): - self.__atime = time() - return getattr(self.__data,attr) diff --git a/primer/13/time60.py b/primer/13/time60.py deleted file mode 100644 index a61ca1e..0000000 --- a/primer/13/time60.py +++ /dev/null @@ -1,14 +0,0 @@ -class Time60(object): - def __init__(self,hr,min): - self.hr = hr - self.min = min - def __str__(self): - return "%d:%d" %(self.hr,self.min) - __repr__ = __str__ - - def __add__(self,other): - return self.__class__(self.hr + other.hr,self.min+other.min) - def __iadd__(self,other): - self.hr += other.hr - self.min += other.min - return self diff --git a/primer/16/sockCli.py b/primer/16/sockCli.py deleted file mode 100644 index f97acee..0000000 --- a/primer/16/sockCli.py +++ /dev/null @@ -1,20 +0,0 @@ -from socket import * - -HOST ="localhost" -PORT = 8888 -BUFSIZ = 1024 -ADDR = (HOST,PORT) - -while True: - tcpCliSock = socket(AF_INET,SOCK_STREAM) - tcpCliSock.connect(ADDR) - data=raw_input(">") - - if not data: - break - tcpCliSock.send("%s\r\n" % data) - data = tcpCliSock.recv(BUFSIZ) - if not data: - break - print data.strip() - tcpCliSock.close() diff --git a/primer/16/sockServ.py b/primer/16/sockServ.py deleted file mode 100644 index 9e4e38f..0000000 --- a/primer/16/sockServ.py +++ /dev/null @@ -1,17 +0,0 @@ -from SocketServer import (TCPServer as TCP, - StreamRequestHandler as SRH) -from time import ctime - -HOST='' -PORT=8888 -ADDR=(HOST,PORT) - -class MyRequestHandler(SRH): - def handle(self): - print "......connected from :",self.client_address - self.wfile.write('[%s] %s' % - (ctime(),self.rfile.readline())) - -tcpSer = TCP(ADDR,MyRequestHandler) -print "waiting for connection..." -tcpSer.serve_forever() diff --git a/primer/16/tcpclient.py b/primer/16/tcpclient.py deleted file mode 100644 index b346fc6..0000000 --- a/primer/16/tcpclient.py +++ /dev/null @@ -1,21 +0,0 @@ -from socket import * - -HOST = "localhost" -PORT = 21567 -BUFSIZ = 1024 -ADDR=(HOST,PORT) - -tcpCliSock = socket(AF_INET,SOCK_STREAM) -tcpCliSock.connect(ADDR) - -while True: - data = raw_input(">") - if not data: - break - tcpCliSock.send(data) - data = tcpCliSock.recv(BUFSIZ) - if not data: - break - print data - -tcpCliSock.close() diff --git a/primer/16/tcpserver.py b/primer/16/tcpserver.py deleted file mode 100644 index 688b0aa..0000000 --- a/primer/16/tcpserver.py +++ /dev/null @@ -1,26 +0,0 @@ - -from socket import * -from time import ctime - -HOST ='' -PORT=21567 -BUFSIZ = 1024 -ADDR =(HOST,PORT) - -tcpSerSock = socket(AF_INET,SOCK_STREAM) -tcpSerSock.bind(ADDR) -tcpSerSock.listen(5) - -while True: - print "waiting from connection..." - tcpCliSock,addr = tcpSerSock.accept() - print "...connected from :",addr - - while True: - data = tcpCliSock.recv(BUFSIZ) - if not data: - break - tcpCliSock.send('[%s] %s' %(ctime(),data)) - - tcpCliSock.close() -tcpSerSock.close() diff --git a/primer/16/udpCli.py b/primer/16/udpCli.py deleted file mode 100644 index 9a502f3..0000000 --- a/primer/16/udpCli.py +++ /dev/null @@ -1,19 +0,0 @@ -from socket import * - -HOST="localhost" -PORT=21567 -BUFSIZ = 1024 -ADDR= (HOST,PORT) - -udpCliSock= socket(AF_INET,SOCK_DGRAM) - -while True: - data = raw_input(">") - if not data: - break - udpCliSock.sendto(data,ADDR) - data,ADDR= udpCliSock.recvfrom(BUFSIZ) - if not data: - break - print data -udpCliSock.close() diff --git a/primer/16/udpSer.py b/primer/16/udpSer.py deleted file mode 100644 index bf7eabb..0000000 --- a/primer/16/udpSer.py +++ /dev/null @@ -1,17 +0,0 @@ -from socket import * -from time import ctime - -HOST ='' -PORT=21567 -BUFSIZ=1024 -ADDR =(HOST,PORT) - -udpSerSock = socket(AF_INET,SOCK_DGRAM) -udpSerSock.bind(ADDR) - -while True: - print "waiting for message..." - data,addr = udpSerSock.recvfrom(BUFSIZ) - udpSerSock.sendto('[%s] %s' %(ctime(),data),addr) - print "...received from and returned to :",addr -udpSerSock.close() diff --git a/primer/18/sleep.py b/primer/18/sleep.py deleted file mode 100644 index 65c80cd..0000000 --- a/primer/18/sleep.py +++ /dev/null @@ -1,26 +0,0 @@ -from time import sleep,ctime -import thread - -loops =[4,2,11] -def loop(nloop,nsec,lock): - print " start loop:",nloop,"at",ctime() - sleep(nsec) - print "loop :",nloop, "done at:",ctime() - -def main(): - print "start at:", ctime() - locks = [] - nloops = range(len(loops)) - - for i in nloops: - lock = thread.allocate_lock() - lock.acquire() - locks.append(lock) - for i in nloops: - thread.start_new_thread(loop,(i,loops[i],locks[i])) - for i in nloops: - while locks[i].locked():pass - - print " all done at:" ,ctime() -if __name__ == "__main__": - main() diff --git a/primer/18/threading.py b/primer/18/threading.py deleted file mode 100644 index 367fcf3..0000000 --- a/primer/18/threading.py +++ /dev/null @@ -1,37 +0,0 @@ -import threading -from time import sleep,ctime - -loops = [4,2] - -class ThreadFunc(object): - def __init__(self,func,args,name=""): - self.name = name - self.name = func - self.args = args - def __call__(self): - apply(self.func,slef.args) - - def loop(nloop,nsec): - print "start loop:",nloop,"at:",ctime() - sleep(nsec) - print "loop",nloop,"done at:",ctime() - - def main(): - print "starting at:",ctime() - threads = [] - nloops = range(len(loops)) - #创建线程 - for i in nloops: - t = threading.Thread(target = loop,args=(i,loops[i])) - threads.append(t); - #启动线程 - for i in nloops: - threads[i].start() - #等待线程 - for i in nloops: - threads[i].join() - print "all DONE at:",ctime() - -if __name__　== "__main__": - main() - diff --git a/primer/19/label.py b/primer/19/label.py deleted file mode 100644 index cec761f..0000000 --- a/primer/19/label.py +++ /dev/null @@ -1,11 +0,0 @@ -import Tkinter - -top = Tkinter.Tk() - -label = Tkinter.Label(top,text="input") -label.pack() - -button = Tkinter.Button(top,text="QIUT",command=top.quit,bg="red",fg="white") -button.pack(fill=Tkinter.X,expand=1) - -Tkinter.mainloop() diff --git a/primer/2/2_10.py b/primer/2/2_10.py deleted file mode 100644 index d2dec79..0000000 --- a/primer/2/2_10.py +++ /dev/null @@ -1,10 +0,0 @@ -a = 19 -print "please input a int number in 1-100" -x = raw_input() -while x != a: - print "sorry error: \n input again:" - x = raw_input() -if x == a: - print "you are right" - break -raw_input() diff --git a/primer/2/2_11.py b/primer/2/2_11.py deleted file mode 100644 index 5c3877d..0000000 --- a/primer/2/2_11.py +++ /dev/null @@ -1,25 +0,0 @@ -print "Menu: input the choice\n" -print "s: sum" -print "a: avg" -print "x: exit" -aa =[1,2,3,4] -while True: - ch = raw_input() - if ch == 's': - i = 0 - Sum = 0 - print "you choice sum:\n" - for i in aa: - Sum += i - print Sum - if ch == 'a': - print "you choice avg:\n" - i = 0 - Sum1 = 0 - for i in aa: - Sum1 += i - avg = float(Sum1)/(len(aa)) - print avg - if ch == 'x': - break - diff --git a/primer/2/2_2.py b/primer/2/2_2.py deleted file mode 100644 index 5d71d06..0000000 --- a/primer/2/2_2.py +++ /dev/null @@ -1,9 +0,0 @@ -a = '19' -print "please input a int number in 1-100" -x = raw_input() -if a != x: - print "sorry you are wrong!\ninput again:" - x = raw_input() -else - print "you are right" -raw_input() diff --git a/primer/2/2_7.py b/primer/2/2_7.py deleted file mode 100644 index 8405c4b..0000000 --- a/primer/2/2_7.py +++ /dev/null @@ -1,5 +0,0 @@ -s = raw_input() -i = 0 -for i in s: - print i -raw_input() diff --git a/primer/2/2_8.py b/primer/2/2_8.py deleted file mode 100644 index 7ea33df..0000000 --- a/primer/2/2_8.py +++ /dev/null @@ -1,7 +0,0 @@ -s=[1,2,3,4,5,6] -i = 0 -t = 0 -for i in s: - t += i -print float(t)/len(s) -raw_input() diff --git a/primer/20/Cralwer.py b/primer/20/Cralwer.py deleted file mode 100644 index 3afa5d2..0000000 --- a/primer/20/Cralwer.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python - -from sys import argv -from os import makedirs,unlink,sep -from os.path import dirname,exists,isdir,splitext -from string import replace,find,lower -from htmllib import HTMLParser -from urllib import urlretrieve -from urlparse import urlparse,urljoin -from formatter import DumbWriter,AbstractFormatter -from cStringIO import StringIO - -class Retriever(object):#下载网页类 - - def __init__(self,url): - self.url = url - self.file = self.filename(url) - - def filename(self,url,deffile ="index.htm"): - parsedurl = urlparse(url,"http:",0) #解析路径 - path = parsedurl[1] + parsedurl[2] - ext = splitext(path) - if ext[1] == "": #如果没有文件，使用默认 - if path[-1] == "/": - path += deffile - else: - path += "/" + deffile - ldir = dirname(path) #本地目录 - if sep != "/": - ldir = replace(ldir,"/",sep) - if not isdir(ldir): #如果没有目录，创建一个 - if exists(ldir):unlink(ldir) - makedirs(ldir) - return path - - def download(self):# 下载网页 - try: - retval = urlretrieve(self.url,self.file) - except IOError: - retval = ('***Error: invalid URL: "%s"' % self.url,) - return retval - - def parseAndGetLinks(self): #解析HTML，保存链接 - self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) - self.parser.feed(open(self.file).read()) - self.parser.close() - return self.parser.anchorlist - - -class Crawler(object): #管理类,管理整个爬行过程 - - count = 0 #下载网页计数器 - - def __init__ (self,url): - self.q = [url] #链接队列 - self.seen = [] #已下载 - self.dom = urlparse(url)[1] #判断链接是否为主链接的子域名 - - def getPage(self,url): #下载网页 - r = Retriever(url) - retval = r.download() - if retval[0] == "*": #错误，不解析 - print retval,"--- skipping parse" - return - Crawler.count += 1 - print '\n(',Crawler.count,')' - print "URL:",url - print "FILE:",retval[0] - self.seen.append(url) - - links = r.parseAndGetLinks() #得到链接 - for eachLink in links: - if eachLink[:4] != "http" and find(eachLink,"://") == -1: - eachLink = urljoin(url,eachLink) - print "* ",eachLink - - if find(lower(eachLink),"mailto:") != -1: #过滤邮箱链接 - print "--- discarded,mailto link" - continue - - if eachLink not in self.seen: - if find(eachLink,self.dom) == -1: - print "---discarded,not in domain" - else: - if eachLink not in self.q: - self.q.append(eachLink) - print "---new,add to Q" - else: - print "---discarded,already in Q" - else: - print "---discarded, arlready processed" - - def go(self): #在队列里处理链接，启动 - while self.q: - url=self.q.pop() - self.getPage(url) - -def main(): - if len(argv) > 1: - url = argv[1] - - else: - try: - url = raw_input("Enter starting URL:") - except (KeyboardInterrupt,EOFError): - url = "" - - if not url: return - robot = Crawler(url) - robot.go() - -if __name__ == "__main__": - main() - diff --git a/primer/3/makeFile.py b/primer/3/makeFile.py deleted file mode 100644 index bee369a..0000000 --- a/primer/3/makeFile.py +++ /dev/null @@ -1,14 +0,0 @@ -import os - -filename=raw_input("filename:") -if os.path.exists(filename): - print "file exist" - exit() - - -fd=open(filename,'r') -fd.writelines(all) -fd.close() - -print "done" - diff --git a/primer/3/makeTextFile.py b/primer/3/makeTextFile.py deleted file mode 100644 index 6969f8f..0000000 --- a/primer/3/makeTextFile.py +++ /dev/null @@ -1,32 +0,0 @@ - -'makeTextFile.py -- create text file' - -import os -ls = os.linesep - -#get filename -while True: - fname = raw_input() - if os.path.exists(fname): - print "ERROR: '%s' already exists " %fname - else: - break - -#get file content lines -all = [] -print "\n Enter lines:('.' by itself to quit)\n" - -#loop until user terminates input -while True: - entry = raw_input('>') - if entry == '.': - break - else: - all.append(entry) - -#write lines to file with proper line-ending -fobj = open(fname,'w') -fobj.writelines(['%s%s' %(x,ls) for x in all]) -fobj.close() -print 'DONE!' - diff --git a/primer/3/readTextFile.py b/primer/3/readTextFile.py deleted file mode 100644 index 980d3e0..0000000 --- a/primer/3/readTextFile.py +++ /dev/null @@ -1,17 +0,0 @@ - -'readTextFile.py -- read and display text file' - -#get filename -fname = raw_input("enter the filename:") -print - -#attempt to open file for reading -try: - fobj = open(fname,'r') -except IOError,e: - print "*** file open error",e -else: - #display contents to the screen - for echoLine in fobj: - print echoLine; -fobj.close() diff --git a/primer/4/display.py b/primer/4/display.py deleted file mode 100644 index e50f368..0000000 --- a/primer/4/display.py +++ /dev/null @@ -1,11 +0,0 @@ -def displayType(num): - print num, 'is', - if isinstance(num,(int,long,float,complex)): - print "a number of type:",type(num).__name__ - else: - print "not a number at all" - -displayType(2) -displayType(-1.9) -displayType(-1+1.9j) -displayType('xcxx') diff --git a/primer/6/idcheck.py b/primer/6/idcheck.py deleted file mode 100644 index b6d6220..0000000 --- a/primer/6/idcheck.py +++ /dev/null @@ -1,20 +0,0 @@ -import string - -alphas = string.letters +'_' -nums = string.digits - -print "Welcom to the indetifter Checker V1.0" -print "Test must be at least 2 Chars long" -myinput = raw_input("Iddentifter to test:\n") - -if len(myinput) > 1: - if myinput[0] not in alphas: - print "invald: first symbol must be alphbetic" - else: - for otherChar in myinput[1:]: - if alphas not in alphas + nums: - print "invalid:remaining symbols must be alphanumeric" - break - else: - print "ok as an identifier" -raw_input() diff --git a/primer/6/queue.py b/primer/6/queue.py deleted file mode 100644 index 7eb364d..0000000 --- a/primer/6/queue.py +++ /dev/null @@ -1,38 +0,0 @@ - -queue = [] -def enQ(): - queue.append(raw_input("enter new string:").strip()) -def deQ(): - if len(queue) == 0: - print "empty queue\n" - else: - print 'Removed![',`queue.pop()`,']' -def viewQ(): - print queue - -CMDS = {'e': enQ,'d':deQ,'v':viewQ} -def showmenu(): - pr=""" -(E)nqueue -(D)enqueue -(V)iew -(Q)uit - - enter choice: """ - while True: - while True: - try: - choice = raw_input(pr).strip()[0].lower() - except (EOFError,KeyboardInterpt,IndexError): - choice = "q" - print "\nYou picked:[%s]" % choice - if choice not in "devq": - print "error,try again" - else: - break - if choice == "q": - break - CMDS[choice]() -if __name__ == '__main__': - showmenu() - diff --git a/primer/6/stack.py b/primer/6/stack.py deleted file mode 100644 index 60dc053..0000000 --- a/primer/6/stack.py +++ /dev/null @@ -1,43 +0,0 @@ -stack = [] - -def pushit(): - stack.append(raw_input("enter a string:").strip()) - -def popit(): - if len(stack) == 0: - print "stack is empty!\n" - else: - print 'Removed![',`stack.pop()`,']' - -def viewstack(): - print stack - -CMDS ={'u': pushit,'o':popit,'v':viewstack} - -def showmenu(): - pr = """ - p(U)sh - p(O)p - (V)iew - (Q)uit - - enter the choice: """ - while True: - while True: - try: - choice = raw_input(pr).strip()[0].lower() - except (EOFError,KeyboardInterrupt,IndexError): - choice ="q" - - print "\nYou picked:[%s]" % choice - if choice not in "uovq": - print "Invalid option,try again" - else: - break - - if choice == "q": - break - CMDS[choice]() -if __name__ == "__main__": - showmenu() - diff --git a/primer/6/unicode.txt b/primer/6/unicode.txt deleted file mode 100644 index f3e8683..0000000 --- a/primer/6/unicode.txt +++ /dev/null @@ -1 +0,0 @@ -hhhh diff --git a/primer/6/unicodeFile.py b/primer/6/unicodeFile.py deleted file mode 100644 index 0752725..0000000 --- a/primer/6/unicodeFile.py +++ /dev/null @@ -1,20 +0,0 @@ -''' -An example for unicode string -''' - -CODEC = "utf-8" -FILE = "unicode.txt" - -hello = u"hhhh\n" -byte = hello.encode(CODEC) -f = open(FILE,"w") -f.write(byte) -f.close() - -f = open(FILE,"r") -byte = f.read() -f.close() - -hello = byte.decode(CODEC) -print hello -raw_input() diff --git a/primer/7/userpw.py b/primer/7/userpw.py deleted file mode 100644 index ebe358e..0000000 --- a/primer/7/userpw.py +++ /dev/null @@ -1,53 +0,0 @@ -db = {} - -def newuser(): - prompt = "login desired:" - while True: - name = raw_input(prompt) - if db.has_key(name): - prompt = "name taken,try again: " - continue - else: - break - pwd = raw_input("passwd:") - db[name]= pwd - print "regeisted oK!\n" - -def olduser(): - name = raw_input("login:") - pwd = raw_input("passwd:") - passwd = db.get(name) - if passwd == pwd: - print "welcome back," ,name - else: - print "login incorrect" - -def showmenu(): - prompt = """ - (n) new user login - (l) exiting user login - (q) quit - enter choice : """ - - done = False - while not done: - chosen = False - while not chosen: - try: - choice = raw_input(prompt).strip()[0].lower() - except(EOFError,KeyboardInterrupt): - choice = "q" - print "\n you picked [%s]" % choice - if choice not in "nlq": - print "invalid option, try again" - else: - chosen = True - - if choice == "q": done = True - if choice == "n": newuser() - if choice == "l": olduser() - -if __name__ == "__main__": - showmenu() - - diff --git a/primer/9/ospathex.py b/primer/9/ospathex.py deleted file mode 100644 index a333211..0000000 --- a/primer/9/ospathex.py +++ /dev/null @@ -1,38 +0,0 @@ - -import os -for tmpdir in("/tmp"): - if os.path.isdir(tmpdir): - break - else: - print "no temp dir available" - tmpdir = "" -if tmpdir: - os.chdir(tmpdir) - cwd = os.getcwd() - print " current tmp dir" - print cwd - - print "create example dir" - os.mkdir("example") - os.chdir("example") - cwd = os.getcwd() - print "new work dir" - print cwd - print "list the dir" - print os.listdir(cwd) - - print "create test file" - fobj = open("test","w") - fobj.write("xxg\n") - fobj.write("111111") - fobj.close() - - print "update the list dir" - print os.listdir(cwd) - - print "rename the file" - os.rename("test","xxg.txt") - print os.listdir(cwd) - - - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..68ffc92 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.pytest.ini_options] +addopts = "-q" +testpaths = ["tests"] diff --git a/requirements-dev.lock.txt b/requirements-dev.lock.txt new file mode 100644 index 0000000..353ef54 --- /dev/null +++ b/requirements-dev.lock.txt @@ -0,0 +1,6 @@ +-r requirements.lock.txt +iniconfig==2.3.0 +packaging==26.3 +pluggy==1.6.0 +Pygments==2.21.0 +pytest==9.1.1 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..1441c92 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest==9.1.1 diff --git a/requirements.lock.txt b/requirements.lock.txt new file mode 100644 index 0000000..bbdff30 --- /dev/null +++ b/requirements.lock.txt @@ -0,0 +1,6 @@ +PyYAML==6.0.3 +certifi==2026.7.22 +charset-normalizer==3.5.1 +idna==3.19 +requests==2.32.5 +urllib3==2.6.3 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4d2c6a5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +PyYAML==6.0.3 +requests==2.32.5 +urllib3==2.6.3 diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..3b1950c --- /dev/null +++ b/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://python.flypython.com/sitemap.xml diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9d9d70d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from datetime import date +from typing import Any + +import pytest + + +@pytest.fixture +def valid_catalog() -> dict[str, Any]: + paths = [] + resources = [] + for order, path_id in enumerate( + ("foundations", "web-apis", "automation", "ai-agents"), start=1 + ): + paths.append( + { + "id": path_id, + "title_en": f"{path_id} title", + "title_zh": f"{path_id} 标题", + "summary_en": f"{path_id} summary", + "summary_zh": f"{path_id} 摘要", + "order": order, + } + ) + resources.append( + { + "id": f"resource-{order}", + "path": path_id, + "title": f"Resource {order}", + "url": f"https://example{order}.com/docs/", + "source_type": "official-docs", + "level": "beginner", + "language": "en", + "why_en": "Primary documentation maintained by the project.", + "why_zh": "由项目维护的官方文档。", + "reviewed_on": date(2026, 8, 31), + "status": "active", + "requires_key": False, + "risk": "low", + "featured": order == 1, + } + ) + return { + "catalog": { + "reviewed_on": date(2026, 8, 31), + "status": "active", + "paths": paths, + }, + "resources": resources, + } diff --git a/tests/render_security_test.rb b/tests/render_security_test.rb new file mode 100644 index 0000000..cefb831 --- /dev/null +++ b/tests/render_security_test.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require "jekyll" +require "tmpdir" + +source = File.expand_path("..", __dir__) + +Dir.mktmpdir("flypython-render-security") do |destination| + config = Jekyll.configuration( + "source" => source, + "destination" => destination, + "quiet" => true, + "disable_disk_cache" => true + ) + site = Jekyll::Site.new(config) + site.reset + site.read + + path = site.data.fetch("resources").fetch("catalog").fetch("paths").first + path["title_en"] = "" + path["title_zh"] = "" + path["summary_en"] = "" + path["summary_zh"] = "" + + resource = site.data.fetch("resources").fetch("resources").first + resource["title"] = "" + resource["why_en"] = "" + resource["why_zh"] = "" + resource["url"] = 'https://example.com/" onmouseover="alert(5)' + + site.generate + site.render + site.cleanup + site.write + + html = ["index.html", File.join("zh", "index.html")].map do |relative_path| + File.read(File.join(destination, relative_path), encoding: "UTF-8") + end.join("\n") + + forbidden = [ + "", + "", + "", + "", + 'href="https://example.com/" onmouseover="alert(5)"' + ] + leaked = forbidden.select { |payload| html.include?(payload) } + abort "unescaped catalog payloads: #{leaked.join(', ')}" unless leaked.empty? + + expected = [ + "<img src=x onerror=alert(1)>", + "<svg onload=alert(2)>", + "</a><script>alert(3)</script>", + "<math href=x onmouseover=alert(4)>", + "https://example.com/" onmouseover="alert(5)" + ] + missing = expected.reject { |payload| html.include?(payload) } + abort "escaped catalog payloads missing: #{missing.join(', ')}" unless missing.empty? +end + +puts "catalog output escaping valid" diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 0000000..eca9134 --- /dev/null +++ b/tests/test_catalog.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from copy import deepcopy +from datetime import date + +import pytest + +from tools.catalog import ( + CatalogLoadError, + canonical_hostname, + load_catalog, + normalize_url, + validate_catalog, +) +from tools.validate_catalog import run + + +def test_valid_catalog_passes(valid_catalog: dict) -> None: + assert validate_catalog(valid_catalog, today=date(2026, 8, 31)) == [] + + +@pytest.mark.parametrize("language", ["en", "zh", "multilingual"]) +def test_supported_language_values_pass(valid_catalog: dict, language: str) -> None: + data = deepcopy(valid_catalog) + data["resources"][0]["language"] = language + assert validate_catalog(data, today=date(2026, 8, 31)) == [] + + +def test_loader_rejects_duplicate_yaml_keys(tmp_path) -> None: + catalog = tmp_path / "resources.yml" + catalog.write_text("catalog: {}\ncatalog: {}\nresources: []\n", encoding="utf-8") + with pytest.raises(CatalogLoadError, match="duplicate key"): + load_catalog(catalog) + + +def test_loader_rejects_non_string_mapping_keys(tmp_path) -> None: + catalog = tmp_path / "resources.yml" + catalog.write_text("? [catalog]\n: {}\nresources: []\n", encoding="utf-8") + with pytest.raises(CatalogLoadError, match="mapping keys must be strings"): + load_catalog(catalog) + + +def test_schema_duplicate_https_date_and_parity_errors(valid_catalog: dict) -> None: + data = deepcopy(valid_catalog) + data["resources"][0]["url"] = "http://example.invalid/docs" + data["resources"][1]["url"] = "https://example.invalid/docs/" + data["resources"][2]["url"] = "https://example.invalid/docs" + data["resources"][2]["reviewed_on"] = "2025-01-01" + data["resources"][3]["why_zh"] = "" + issues = validate_catalog(data, today=date(2026, 8, 31), max_review_age_days=366) + codes = {issue.code for issue in issues} + assert {"https-required", "duplicate-url", "stale-review", "invalid-text"} <= codes + + +def test_resource_ids_and_urls_reject_unsafe_forms(valid_catalog: dict) -> None: + data = deepcopy(valid_catalog) + data["resources"][0]["id"] = "Not A Slug" + data["resources"][0]["url"] = "https://user:secret@example.com/docs" + codes = { + issue.code for issue in validate_catalog(data, today=date(2026, 8, 31)) + } + assert {"invalid-id", "url-credentials"} <= codes + + +def test_path_orders_must_be_consecutive(valid_catalog: dict) -> None: + data = deepcopy(valid_catalog) + data["catalog"]["paths"][-1]["order"] = 5 + + codes = { + issue.code for issue in validate_catalog(data, today=date(2026, 8, 31)) + } + + assert "order-parity" in codes + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + ( + "HTTPS://[2001:4860:4860::8888]:443/docs/", + "https://[2001:4860:4860::8888]/docs", + ), + ( + "https://[2001:4860:4860::8888]:8443/docs/", + "https://[2001:4860:4860::8888]:8443/docs", + ), + ], +) +def test_normalize_url_preserves_ipv6_brackets(url: str, expected: str) -> None: + assert normalize_url(url) == expected + + +def test_hostname_canonicalization_handles_idna_and_trailing_dot() -> None: + assert canonical_hostname("BÜCHER.example.") == "xn--bcher-kva.example" + + +def test_validator_detects_idna_equivalent_duplicate_urls(valid_catalog: dict) -> None: + data = deepcopy(valid_catalog) + data["resources"][0]["url"] = "https://bücher.example/docs/" + data["resources"][1]["url"] = "https://xn--bcher-kva.example/docs" + + codes = { + issue.code for issue in validate_catalog(data, today=date(2026, 8, 31)) + } + + assert "duplicate-url" in codes + + +def test_validator_reports_non_string_mapping_keys(valid_catalog: dict) -> None: + data = deepcopy(valid_catalog) + data["catalog"][1] = "unexpected" + + issues = validate_catalog(data, today=date(2026, 8, 31)) + + assert any(issue.code == "invalid-key" for issue in issues) + + +def test_validator_exit_code_for_invalid_catalog(tmp_path) -> None: + catalog = tmp_path / "resources.yml" + catalog.write_text("catalog: {}\nresources: []\n", encoding="utf-8") + assert run(["--catalog", str(catalog)]) == 1 diff --git a/tests/test_check_links.py b/tests/test_check_links.py new file mode 100644 index 0000000..f129efb --- /dev/null +++ b/tests/test_check_links.py @@ -0,0 +1,678 @@ +from __future__ import annotations + +import json +import threading +from pathlib import Path + +import pytest +import requests +import tools.check_links as check_links + +from tools.check_links import ( + CatalogLink, + LinkChecker, + LinkResult, + PinnedDNSHTTPAdapter, + SafeTargetGuard, + ThreadLocalSessions, + UnsafeTarget, + build_report, + build_session, + classify_status, + exit_code_for_report, + select_links, +) + + +PUBLIC_IP = "93.184.216.34" + + +class FakeResponse: + def __init__(self, status_code, url="https://example.com/", history=None, headers=None): + self.status_code = status_code + self.url = url + self.history = history or [] + self.headers = headers or {} + self.closed = False + + def close(self): + self.closed = True + + +class FakeSession: + def __init__(self, head, get=None): + self.head_response = head + self.get_response = get + self.calls = [] + self.closed = False + + def head(self, url, **kwargs): + self.calls.append(("HEAD", url, kwargs)) + return self.head_response + + def get(self, url, **kwargs): + self.calls.append(("GET", url, kwargs)) + return self.get_response + + def close(self): + self.closed = True + + +def guard_for(address=PUBLIC_IP): + return SafeTargetGuard(lambda _host, _port: [address]) + + +def link(): + return CatalogLink("docs", "foundations", "Docs", "https://example.com/docs") + + +@pytest.mark.parametrize( + ("status_code", "expected"), + [ + (200, "working"), + (301, "redirect"), + (403, "review"), + (408, "review"), + (425, "review"), + (429, "review"), + (503, "review"), + (404, "broken"), + ], +) +def test_status_classification(status_code, expected) -> None: + assert classify_status(status_code, redirected=False) == expected + + +def test_redirect_history_is_preserved() -> None: + hop = FakeResponse(301, "https://example.com/old", headers={"Location": "/docs"}) + response = FakeResponse(200, "https://example.com/docs") + + class RedirectSession(FakeSession): + def __init__(self): + super().__init__(hop) + self.responses = iter([hop, response]) + + def head(self, url, **kwargs): + self.calls.append(("HEAD", url, kwargs)) + return next(self.responses) + + session = RedirectSession() + checker = LinkChecker( + guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0 + ) + redirect_link = CatalogLink( + "docs", "foundations", "Docs", "https://example.com/old" + ) + result = checker.check_one(redirect_link) + assert result.status == "redirect" + assert result.history == [ + {"status_code": 301, "url": "https://example.com/old", "location": "/docs"} + ] + assert hop.closed is True + + +def test_head_failure_falls_back_to_streaming_get_and_confirms_404() -> None: + session = FakeSession(FakeResponse(404), FakeResponse(404)) + checker = LinkChecker( + guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0 + ) + result = checker.check_one(link()) + assert result.status == "broken" + assert result.method == "GET" + assert session.calls[1][2]["stream"] is True + + +def test_head_redirect_without_location_falls_back_to_get() -> None: + head = FakeResponse(301) + response = FakeResponse(200) + session = FakeSession(head, response) + checker = LinkChecker( + guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0 + ) + + result = checker.check_one(link()) + + assert result.status == "working" + assert result.method == "GET" + assert head.closed is True + assert response.closed is True + + +def test_get_redirect_without_location_is_fatal() -> None: + head = FakeResponse(301) + response = FakeResponse(301) + session = FakeSession(head, response) + checker = LinkChecker( + guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0 + ) + + result = checker.check_one(link()) + + assert result.status == "error" + assert "no Location" in (result.error or "") + assert head.closed is True + assert response.closed is True + + +def test_unsupported_redirect_status_is_fatal() -> None: + response = FakeResponse(304) + session = FakeSession(response) + checker = LinkChecker( + guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0 + ) + + result = checker.check_one(link()) + + assert result.status == "error" + assert "unsupported redirect status 304" in (result.error or "") + assert response.closed is True + + +@pytest.mark.parametrize("status_code", [403, 408, 425, 429, 500, 503]) +def test_transient_and_access_denied_statuses_need_review(status_code) -> None: + head = FakeResponse(status_code) + session = FakeSession(head, FakeResponse(status_code)) + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: session, + workers=1, + retries=0, + min_interval=0, + ) + assert checker.check_one(link()).status == "review" + assert len(session.calls) == 1 + assert head.closed is True + + +@pytest.mark.parametrize( + "url", + [ + "http://localhost/admin", + "http://127.0.0.1/", + "http://169.254.169.254/latest/meta-data/", + "http://100.100.100.200/latest/meta-data/", + "http://metadata.google.internal/computeMetadata/v1/", + "https://168.63.129.16/", + "https://224.0.0.1/", + "https://[64:ff9b::7f00:1]/", + "https://[::ffff:93.184.216.34]/", + ], +) +def test_literal_local_and_metadata_targets_are_blocked(url) -> None: + with pytest.raises(UnsafeTarget): + SafeTargetGuard().resolve_url(url) + + +def test_dns_resolution_to_private_ip_is_blocked() -> None: + with pytest.raises(UnsafeTarget, match="non-public"): + guard_for("10.0.0.8").resolve_url("https://example.com/") + + +def test_adapter_pins_public_ip_and_preserves_tls_hostname() -> None: + adapter = PinnedDNSHTTPAdapter(guard_for(), max_retries=0) + captured = {} + + class PoolManager: + def connection_from_host(self, **kwargs): + captured.update(kwargs) + return "pool" + + adapter.poolmanager = PoolManager() + request = requests.Request("GET", "https://example.com/path").prepare() + assert adapter.get_connection_with_tls_context(request, True) == "pool" + assert captured["host"] == PUBLIC_IP + assert captured["pool_kwargs"]["server_hostname"] == "example.com" + assert captured["pool_kwargs"]["assert_hostname"] == "example.com" + assert request.headers["Host"] == "example.com" + + +def test_adapter_revalidates_and_blocks_an_unsafe_redirect_hop() -> None: + resolved_hosts = [] + + def resolver(host, _port): + resolved_hosts.append(host) + return [PUBLIC_IP if host == "example.com" else "127.0.0.1"] + + adapter = PinnedDNSHTTPAdapter(SafeTargetGuard(resolver), max_retries=0) + + class PoolManager: + def connection_from_host(self, **_kwargs): + return "pool" + + adapter.poolmanager = PoolManager() + first = requests.Request("GET", "https://example.com/start").prepare() + redirect = requests.Request("GET", "https://internal.example/admin").prepare() + + assert adapter.get_connection_with_tls_context(first, True) == "pool" + with pytest.raises(UnsafeTarget, match="non-public"): + adapter.get_connection_with_tls_context(redirect, True) + assert resolved_hosts == ["example.com", "internal.example"] + + +def test_checker_blocks_https_redirect_downgrade() -> None: + hop = FakeResponse( + 302, + "https://example.com/start", + headers={"Location": "http://example.com/docs"}, + ) + session = FakeSession(hop) + checker = LinkChecker( + guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0 + ) + + result = checker.check_one(link()) + + assert result.status == "blocked" + assert "may not downgrade" in (result.error or "") + assert hop.closed is True + + +def test_thread_local_sessions_are_not_shared_between_workers() -> None: + created = [] + barrier = threading.Barrier(2) + + def factory(): + value = object() + created.append(value) + return value + + sessions = ThreadLocalSessions(factory) + results = [] + + def worker(): + first = sessions.get() + barrier.wait() + results.append((first, sessions.get())) + + threads = [threading.Thread(target=worker) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert len(created) == 2 + assert results[0][0] is results[0][1] + assert results[1][0] is results[1][1] + assert results[0][0] is not results[1][0] + + +def test_check_all_closes_worker_sessions() -> None: + sessions = [] + + def factory(): + session = FakeSession(FakeResponse(200)) + sessions.append(session) + return session + + checker = LinkChecker( + guard=guard_for(), session_factory=factory, workers=1, min_interval=0 + ) + assert len(checker.check_all([link()])) == 1 + assert len(sessions) == 1 + assert sessions[0].closed is True + + +def test_mode_selects_internal_external_or_all() -> None: + data = { + "resources": [ + {"id": "internal", "path": "foundations", "title": "I", "url": "/guide/"}, + { + "id": "external", + "path": "foundations", + "title": "E", + "url": "https://docs.python.org/3/", + }, + ] + } + internal = select_links(data, mode="internal", base_url="https://python.flypython.com/") + external = select_links(data, mode="external", base_url="https://python.flypython.com/") + all_links = select_links(data, mode="all", base_url="https://python.flypython.com/") + assert [item.id for item in internal] == ["internal"] + assert [item.id for item in external] == ["external"] + assert len(all_links) == 2 + + +def test_exit_code_fails_every_actionable_result() -> None: + for status in ("review", "broken", "blocked", "error"): + result = LinkResult("id", "path", "Title", "https://example.com", status) + report = build_report(catalog=Path("catalog.yml"), mode="all", results=[result]) + assert exit_code_for_report(report) == 1 + + working = LinkResult( + "id", "path", "Title", "https://example.com", "working" + ) + report = build_report(catalog=Path("catalog.yml"), mode="all", results=[working]) + assert exit_code_for_report(report) == 0 + + +@pytest.mark.parametrize( + "report", + [ + {}, + {"counts": {}}, + {"counts": {"total": 0}}, + { + "counts": { + "working": 1, + "redirect": 0, + "review": 0, + "broken": 0, + "blocked": 0, + "error": 0, + } + }, + { + "counts": { + "working": 1, + "redirect": 0, + "review": 0, + "broken": 0, + "blocked": 0, + "error": 0, + "unknown": 1, + "total": 1, + } + }, + { + "counts": { + "working": 0, + "redirect": 0, + "review": 0, + "broken": 0, + "blocked": 0, + "error": 0, + "total": 1, + } + }, + ], +) +def test_exit_code_rejects_malformed_reports(report: dict) -> None: + assert exit_code_for_report(report) == 2 + + +def test_unknown_result_status_cannot_fail_open() -> None: + result = LinkResult("id", "path", "Title", "https://example.com", "typo") + report = build_report(catalog=Path("catalog.yml"), mode="all", results=[result]) + + assert exit_code_for_report(report) == 2 + + +def test_unexpected_checker_error_is_fatal() -> None: + class BrokenSession(FakeSession): + def head(self, url, **kwargs): + raise RuntimeError("programming defect") + + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: BrokenSession(FakeResponse(200)), + workers=1, + retries=0, + min_interval=0, + ) + result = checker.check_one(link()) + assert result.status == "error" + assert "programming defect" in (result.error or "") + + +@pytest.mark.parametrize( + "exception", + [ + requests.exceptions.ConnectionError("connection failed"), + requests.exceptions.SSLError("certificate failed"), + requests.exceptions.TooManyRedirects("redirect loop"), + requests.exceptions.InvalidURL("invalid redirect"), + ], +) +def test_terminal_request_failures_are_fatal(exception) -> None: + class BrokenSession(FakeSession): + def head(self, url, **kwargs): + raise exception + + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: BrokenSession(FakeResponse(200)), + workers=1, + retries=0, + min_interval=0, + ) + + assert checker.check_one(link()).status == "error" + + +def test_timeout_remains_review_needed() -> None: + class SlowSession(FakeSession): + def head(self, url, **kwargs): + raise requests.Timeout("timed out") + + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: SlowSession(FakeResponse(200)), + workers=1, + retries=0, + min_interval=0, + ) + + assert checker.check_one(link()).status == "review" + + +def test_adapter_transport_and_status_retries_are_disabled() -> None: + session = build_session(guard_for()) + try: + retry = session.get_adapter("https://").max_retries + assert retry.respect_retry_after_header is False + assert retry.backoff_max == 5.0 + assert retry.total == 0 + assert retry.connect == 0 + assert retry.read == 0 + assert retry.status == 0 + assert not retry.status_forcelist + finally: + session.close() + + +def test_status_retry_is_manual_and_closes_each_response() -> None: + first = FakeResponse(503) + second = FakeResponse(200) + + class RetrySession(FakeSession): + def __init__(self): + super().__init__(first) + self.responses = iter([first, second]) + + def head(self, url, **kwargs): + self.calls.append(("HEAD", url, kwargs)) + return next(self.responses) + + session = RetrySession() + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: session, + workers=1, + retries=1, + backoff_factor=0, + min_interval=0, + ) + + result = checker.check_one(link()) + + assert result.status == "working" + assert len(session.calls) == 2 + assert first.closed is True + assert second.closed is True + + +@pytest.mark.parametrize("retry_after", ["60", "9" * 400]) +def test_large_retry_after_stops_without_get_fallback( + monkeypatch, retry_after: str +) -> None: + response = FakeResponse(429, headers={"Retry-After": retry_after}) + session = FakeSession(response) + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: session, + workers=1, + retries=2, + min_interval=0, + ) + sleeps = [] + monkeypatch.setattr(check_links.time, "sleep", sleeps.append) + + result = checker.check_one(link()) + + assert result.status == "review" + assert len(session.calls) == 1 + assert sleeps == [] + assert response.closed is True + + +def test_bounded_retry_after_is_honored(monkeypatch) -> None: + first = FakeResponse(429, headers={"Retry-After": "2"}) + second = FakeResponse(200) + + class RetrySession(FakeSession): + def __init__(self): + super().__init__(first) + self.responses = iter([first, second]) + + def head(self, url, **kwargs): + self.calls.append(("HEAD", url, kwargs)) + return next(self.responses) + + session = RetrySession() + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: session, + workers=1, + retries=1, + min_interval=0, + ) + sleeps = [] + monkeypatch.setattr(check_links.time, "sleep", sleeps.append) + + result = checker.check_one(link()) + + assert result.status == "working" + assert len(session.calls) == 2 + assert sleeps == [2.0] + + +def test_transport_timeout_retry_is_manual() -> None: + response = FakeResponse(200) + + class FlakySession(FakeSession): + def __init__(self): + super().__init__(response) + self.attempt = 0 + + def head(self, url, **kwargs): + self.calls.append(("HEAD", url, kwargs)) + self.attempt += 1 + if self.attempt == 1: + raise requests.Timeout("timed out") + return response + + session = FlakySession() + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: session, + workers=1, + retries=1, + backoff_factor=0, + min_interval=0, + ) + + result = checker.check_one(link()) + + assert result.status == "working" + assert len(session.calls) == 2 + + +def test_wrapped_timeout_remains_review_needed() -> None: + class WrappedTimeoutSession(FakeSession): + def head(self, url, **kwargs): + raise requests.ConnectionError(TimeoutError("timed out")) + + checker = LinkChecker( + guard=guard_for(), + session_factory=lambda: WrappedTimeoutSession(FakeResponse(200)), + workers=1, + retries=0, + min_interval=0, + ) + + assert checker.check_one(link()).status == "review" + + +def test_responses_close_when_result_processing_fails(monkeypatch) -> None: + head = FakeResponse(404) + response = FakeResponse(200) + session = FakeSession(head, response) + checker = LinkChecker( + guard=guard_for(), session_factory=lambda: session, workers=1, min_interval=0 + ) + + def fail_classification(_status_code, *, redirected): + raise RuntimeError("cannot process response") + + monkeypatch.setattr(check_links, "classify_status", fail_classification) + result = checker.check_one(link()) + + assert result.status == "error" + assert head.closed is True + assert response.closed is True + + +@pytest.mark.parametrize( + ("option", "value"), + [ + ("--timeout", "nan"), + ("--timeout", "inf"), + ("--backoff", "nan"), + ("--min-interval", "inf"), + ], +) +def test_cli_rejects_non_finite_float_arguments(option: str, value: str) -> None: + with pytest.raises(SystemExit, match="2"): + check_links.build_parser().parse_args([option, value]) + + +@pytest.mark.parametrize( + ("status", "status_code", "expected_exit"), + [ + ("review", 408, 1), + ("broken", 404, 1), + ("blocked", None, 1), + ("error", None, 1), + ], +) +def test_cli_writes_json_and_uses_report_exit_code( + monkeypatch, + tmp_path: Path, + valid_catalog: dict, + status: str, + status_code: int | None, + expected_exit: int, +) -> None: + monkeypatch.setattr(check_links, "load_catalog", lambda _path: valid_catalog) + monkeypatch.setattr(check_links, "validate_catalog", lambda _data: []) + result = LinkResult( + "docs", + "foundations", + "Docs", + "https://example.com/docs", + status, + status_code=status_code, + method="GET" if status_code is not None else None, + ) + monkeypatch.setattr( + check_links.LinkChecker, "check_all", lambda _self, _links: [result] + ) + output = tmp_path / f"{status}.json" + + exit_code = check_links.run( + ["--catalog", "ignored.yml", "--output", str(output), "--min-interval", "0"] + ) + + assert exit_code == expected_exit + report = json.loads(output.read_text(encoding="utf-8")) + assert report["counts"][status] == 1 + assert report["results"][0]["status_code"] == status_code diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..1d51398 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ +"""Maintenance tooling for the FlyPython resource catalog.""" diff --git a/tools/catalog.py b/tools/catalog.py new file mode 100644 index 0000000..65e5111 --- /dev/null +++ b/tools/catalog.py @@ -0,0 +1,432 @@ +"""Loading and validation helpers for ``_data/resources.yml``.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import date, datetime +from pathlib import Path +from typing import Any, Mapping, Sequence +from urllib.parse import urlsplit, urlunsplit + +import yaml +from yaml.constructor import ConstructorError +from yaml.resolver import BaseResolver + + +EXPECTED_PATH_IDS = {"foundations", "web-apis", "automation", "ai-agents"} +CATALOG_KEYS = {"reviewed_on", "status", "paths"} +PATH_KEYS = {"id", "title_en", "title_zh", "summary_en", "summary_zh", "order"} +RESOURCE_KEYS = { + "id", + "path", + "title", + "url", + "source_type", + "level", + "language", + "why_en", + "why_zh", + "reviewed_on", + "status", + "requires_key", + "risk", + "featured", +} +SOURCE_TYPES = {"official-docs", "official-standard", "official-project"} +LEVELS = {"beginner", "intermediate", "advanced", "all-levels"} +LANGUAGES = {"en", "zh", "multilingual"} +STATUSES = {"active"} +RISKS = {"low", "medium"} +ID_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + + +class CatalogLoadError(ValueError): + """Raised when the catalog cannot be parsed safely.""" + + +class UniqueKeyLoader(yaml.SafeLoader): + """Safe YAML loader that rejects duplicate mapping keys.""" + + +def _construct_unique_mapping( + loader: UniqueKeyLoader, node: yaml.MappingNode, deep: bool = False +) -> dict[Any, Any]: + mapping: dict[Any, Any] = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if not isinstance(key, str): + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "mapping keys must be strings", + key_node.start_mark, + ) + if key in mapping: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +UniqueKeyLoader.add_constructor( + BaseResolver.DEFAULT_MAPPING_TAG, _construct_unique_mapping +) + + +@dataclass(frozen=True) +class ValidationIssue: + code: str + location: str + message: str + + def as_dict(self) -> dict[str, str]: + return { + "code": self.code, + "location": self.location, + "message": self.message, + } + + +def load_catalog(path: str | Path) -> dict[str, Any]: + catalog_path = Path(path) + try: + with catalog_path.open("r", encoding="utf-8") as handle: + value = yaml.load(handle, Loader=UniqueKeyLoader) + except (OSError, yaml.YAMLError) as exc: + raise CatalogLoadError(f"cannot load {catalog_path}: {exc}") from exc + if not isinstance(value, dict): + raise CatalogLoadError(f"{catalog_path} must contain a YAML mapping") + return value + + +def canonical_hostname(value: str) -> str: + """Return the lowercase IDNA form used for URL equality and host buckets.""" + + return value.rstrip(".").encode("idna").decode("ascii").lower() + + +def normalize_url(value: str) -> str: + parsed = urlsplit(value.strip()) + host = canonical_hostname(parsed.hostname or "") + port = parsed.port + default_port = (parsed.scheme.lower() == "https" and port == 443) or ( + parsed.scheme.lower() == "http" and port == 80 + ) + display_host = f"[{host}]" if ":" in host else host + netloc = display_host + if port and not default_port: + netloc = f"{display_host}:{port}" + path = parsed.path or "/" + if path != "/": + path = path.rstrip("/") + return urlunsplit((parsed.scheme.lower(), netloc, path, parsed.query, "")) + + +def _date_value(value: Any) -> date | None: + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + if isinstance(value, str): + try: + return date.fromisoformat(value) + except ValueError: + return None + return None + + +def _missing_or_unknown( + value: Mapping[str, Any], expected: set[str], location: str +) -> list[ValidationIssue]: + issues: list[ValidationIssue] = [] + string_keys = {key for key in value if isinstance(key, str)} + for key in value: + if not isinstance(key, str): + issues.append( + ValidationIssue( + "invalid-key", + f"{location}[{key!r}]", + "mapping keys must be strings", + ) + ) + for key in sorted(expected - string_keys): + issues.append(ValidationIssue("missing-field", location, f"missing {key!r}")) + for key in sorted(string_keys - expected): + issues.append( + ValidationIssue("unknown-field", f"{location}.{key}", "unknown field") + ) + return issues + + +def _required_text( + value: Mapping[str, Any], key: str, location: str +) -> list[ValidationIssue]: + raw = value.get(key) + if not isinstance(raw, str) or not raw.strip(): + return [ + ValidationIssue( + "invalid-text", f"{location}.{key}", "must be a non-empty string" + ) + ] + return [] + + +def validate_catalog( + data: Mapping[str, Any], *, today: date | None = None, max_review_age_days: int = 366 +) -> list[ValidationIssue]: + """Validate schema, uniqueness, HTTPS, review dates, and bilingual parity.""" + + today = today or date.today() + issues: list[ValidationIssue] = [] + expected_top = {"catalog", "resources"} + issues.extend(_missing_or_unknown(data, expected_top, "$")) + + metadata = data.get("catalog") + if not isinstance(metadata, dict): + issues.append( + ValidationIssue("invalid-type", "$.catalog", "must be a mapping") + ) + metadata = {} + else: + issues.extend(_missing_or_unknown(metadata, CATALOG_KEYS, "$.catalog")) + + catalog_date = _date_value(metadata.get("reviewed_on")) + if catalog_date is None: + issues.append( + ValidationIssue( + "invalid-date", "$.catalog.reviewed_on", "must be YYYY-MM-DD" + ) + ) + elif catalog_date > today: + issues.append( + ValidationIssue( + "future-date", "$.catalog.reviewed_on", "cannot be in the future" + ) + ) + if metadata.get("status") not in STATUSES: + issues.append( + ValidationIssue( + "invalid-enum", "$.catalog.status", "must be 'active'" + ) + ) + + paths = metadata.get("paths") + path_ids: set[str] = set() + path_orders: set[int] = set() + if not isinstance(paths, list): + issues.append( + ValidationIssue("invalid-type", "$.catalog.paths", "must be a list") + ) + paths = [] + for index, path_entry in enumerate(paths): + location = f"$.catalog.paths[{index}]" + if not isinstance(path_entry, dict): + issues.append(ValidationIssue("invalid-type", location, "must be a mapping")) + continue + issues.extend(_missing_or_unknown(path_entry, PATH_KEYS, location)) + for key in ("id", "title_en", "title_zh", "summary_en", "summary_zh"): + issues.extend(_required_text(path_entry, key, location)) + path_id = path_entry.get("id") + if isinstance(path_id, str): + if path_id in path_ids: + issues.append( + ValidationIssue("duplicate-path", f"{location}.id", path_id) + ) + path_ids.add(path_id) + order = path_entry.get("order") + if not isinstance(order, int) or isinstance(order, bool) or order < 1: + issues.append( + ValidationIssue( + "invalid-order", f"{location}.order", "must be a positive integer" + ) + ) + elif order in path_orders: + issues.append( + ValidationIssue("duplicate-order", f"{location}.order", str(order)) + ) + else: + path_orders.add(order) + + if path_ids != EXPECTED_PATH_IDS: + missing = sorted(EXPECTED_PATH_IDS - path_ids) + extra = sorted(path_ids - EXPECTED_PATH_IDS) + issues.append( + ValidationIssue( + "path-parity", + "$.catalog.paths", + f"expected four canonical paths; missing={missing}, extra={extra}", + ) + ) + + expected_path_orders = set(range(1, len(EXPECTED_PATH_IDS) + 1)) + if path_orders != expected_path_orders: + missing = sorted(expected_path_orders - path_orders) + extra = sorted(path_orders - expected_path_orders) + issues.append( + ValidationIssue( + "order-parity", + "$.catalog.paths", + f"expected consecutive path orders 1-{len(EXPECTED_PATH_IDS)}; " + f"missing={missing}, extra={extra}", + ) + ) + + resources = data.get("resources") + if not isinstance(resources, list): + issues.append( + ValidationIssue("invalid-type", "$.resources", "must be a list") + ) + resources = [] + + seen_ids: set[str] = set() + seen_urls: dict[str, str] = {} + resources_per_path = {path_id: 0 for path_id in EXPECTED_PATH_IDS} + for index, resource in enumerate(resources): + location = f"$.resources[{index}]" + if not isinstance(resource, dict): + issues.append(ValidationIssue("invalid-type", location, "must be a mapping")) + continue + issues.extend(_missing_or_unknown(resource, RESOURCE_KEYS, location)) + for key in ("id", "path", "title", "url", "why_en", "why_zh"): + issues.extend(_required_text(resource, key, location)) + + resource_id = resource.get("id") + if isinstance(resource_id, str): + if not ID_PATTERN.fullmatch(resource_id): + issues.append( + ValidationIssue( + "invalid-id", + f"{location}.id", + "must be a lowercase kebab-case identifier", + ) + ) + if resource_id in seen_ids: + issues.append( + ValidationIssue("duplicate-id", f"{location}.id", resource_id) + ) + seen_ids.add(resource_id) + + path_id = resource.get("path") + if isinstance(path_id, str): + if path_id not in path_ids: + issues.append( + ValidationIssue( + "unknown-path", f"{location}.path", f"unknown path {path_id!r}" + ) + ) + if path_id in resources_per_path: + resources_per_path[path_id] += 1 + + url = resource.get("url") + if isinstance(url, str) and url.strip(): + try: + parsed = urlsplit(url) + normalized = normalize_url(url) + except ValueError as exc: + issues.append( + ValidationIssue("invalid-url", f"{location}.url", str(exc)) + ) + else: + if parsed.scheme.lower() != "https" or not parsed.hostname: + issues.append( + ValidationIssue( + "https-required", + f"{location}.url", + "must be an absolute HTTPS URL", + ) + ) + if parsed.username or parsed.password: + issues.append( + ValidationIssue( + "url-credentials", + f"{location}.url", + "must not contain embedded credentials", + ) + ) + previous = seen_urls.get(normalized) + if previous: + issues.append( + ValidationIssue( + "duplicate-url", + f"{location}.url", + f"duplicates {previous}", + ) + ) + else: + seen_urls[normalized] = str(resource_id or location) + + enum_fields = { + "source_type": SOURCE_TYPES, + "level": LEVELS, + "language": LANGUAGES, + "status": STATUSES, + "risk": RISKS, + } + for key, allowed in enum_fields.items(): + if resource.get(key) not in allowed: + issues.append( + ValidationIssue( + "invalid-enum", + f"{location}.{key}", + f"must be one of {sorted(allowed)}", + ) + ) + for key in ("requires_key", "featured"): + if not isinstance(resource.get(key), bool): + issues.append( + ValidationIssue( + "invalid-boolean", f"{location}.{key}", "must be a boolean" + ) + ) + + reviewed_on = _date_value(resource.get("reviewed_on")) + if reviewed_on is None: + issues.append( + ValidationIssue( + "invalid-date", f"{location}.reviewed_on", "must be YYYY-MM-DD" + ) + ) + else: + if reviewed_on > today: + issues.append( + ValidationIssue( + "future-date", + f"{location}.reviewed_on", + "cannot be in the future", + ) + ) + elif (today - reviewed_on).days > max_review_age_days: + issues.append( + ValidationIssue( + "stale-review", + f"{location}.reviewed_on", + f"older than {max_review_age_days} days", + ) + ) + if catalog_date and reviewed_on > catalog_date: + issues.append( + ValidationIssue( + "date-parity", + f"{location}.reviewed_on", + "cannot be newer than catalog.reviewed_on", + ) + ) + + for path_id, count in sorted(resources_per_path.items()): + if count == 0: + issues.append( + ValidationIssue( + "empty-path", "$.resources", f"path {path_id!r} has no resources" + ) + ) + return issues + + +def catalog_resources(data: Mapping[str, Any]) -> Sequence[Mapping[str, Any]]: + resources = data.get("resources") + return resources if isinstance(resources, list) else [] diff --git a/tools/check_links.py b/tools/check_links.py new file mode 100644 index 0000000..d5ef293 --- /dev/null +++ b/tools/check_links.py @@ -0,0 +1,746 @@ +#!/usr/bin/env python3 +"""Safely check links declared in ``_data/resources.yml``.""" + +from __future__ import annotations + +import argparse +import ipaddress +import json +import math +import socket +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Protocol +from urllib.parse import urljoin, urlsplit + +import requests +from requests.adapters import HTTPAdapter +from urllib3.exceptions import TimeoutError as Urllib3TimeoutError +from urllib3.util.retry import Retry + +try: + from tools.catalog import ( + CatalogLoadError, + canonical_hostname, + catalog_resources, + load_catalog, + validate_catalog, + ) +except ModuleNotFoundError: # Direct ``python tools/check_links.py`` execution. + from catalog import ( + CatalogLoadError, + canonical_hostname, + catalog_resources, + load_catalog, + validate_catalog, + ) + + +ROOT_DIR = Path(__file__).resolve().parent.parent +DEFAULT_CATALOG = ROOT_DIR / "_data" / "resources.yml" +DEFAULT_OUTPUT = ROOT_DIR / "reports" / "link_check_results.json" +DEFAULT_BASE_URL = "https://python.flypython.com/" +REVIEW_STATUS_CODES = {403, 408, 425, 429} +RETRY_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504} +REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308} +REPORT_STATUSES = ("working", "redirect", "review", "broken", "blocked", "error") +ACTIONABLE_STATUSES = ("review", "broken", "blocked", "error") +MAX_REDIRECTS = 5 +MAX_BACKOFF_SECONDS = 5.0 +KNOWN_METADATA_HOSTS = { + "instance-data", + "metadata", + "metadata.google.internal", + "metadata.google", +} +KNOWN_METADATA_IPS = { + ipaddress.ip_address("168.63.129.16"), + ipaddress.ip_address("169.254.169.254"), + ipaddress.ip_address("100.100.100.200"), + ipaddress.ip_address("fd00:ec2::254"), +} +BLOCKED_IPV6_TRANSLATION_NETWORKS = ( + ipaddress.ip_network("64:ff9b::/96"), + ipaddress.ip_network("64:ff9b:1::/48"), +) + + +class UnsafeTarget(ValueError): + """Raised before a request can reach a non-public network target.""" + + +class RedirectProtocolError(requests.RequestException): + """Raised when a redirect response cannot be followed safely.""" + + +@dataclass(frozen=True) +class ResolvedTarget: + host: str + port: int + addresses: tuple[str, ...] + + +class Resolver(Protocol): + def __call__(self, host: str, port: int) -> Iterable[str]: ... + + +def _system_resolver(host: str, port: int) -> Iterable[str]: + for family, _, _, _, sockaddr in socket.getaddrinfo( + host, port, type=socket.SOCK_STREAM + ): + if family in (socket.AF_INET, socket.AF_INET6): + yield sockaddr[0] + + +def _is_unsafe_ip(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + if address in KNOWN_METADATA_IPS: + return True + if ( + not address.is_global + or address.is_private + or address.is_loopback + or address.is_link_local + or address.is_multicast + or address.is_reserved + or address.is_unspecified + ): + return True + if isinstance(address, ipaddress.IPv6Address): + if address.ipv4_mapped or address.sixtofour or address.teredo: + return True + if any(address in network for network in BLOCKED_IPV6_TRANSLATION_NETWORKS): + return True + return False + + +def _is_timeout_error(error: BaseException) -> bool: + """Recognize timeouts even when requests wraps urllib3 exceptions.""" + + pending: list[BaseException] = [error] + seen: set[int] = set() + timeout_types = (requests.Timeout, TimeoutError, Urllib3TimeoutError) + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, timeout_types): + return True + for nested in ( + current.__cause__, + current.__context__, + getattr(current, "reason", None), + *current.args, + ): + if isinstance(nested, BaseException): + pending.append(nested) + return False + + +def _retry_delay(response: Any, attempt: int, backoff_factor: float) -> float | None: + """Return a bounded retry delay, or ``None`` when the server asks for longer.""" + + raw_retry_after = response.headers.get("Retry-After") + retry_after: float | None = None + if raw_retry_after: + try: + retry_after = int(raw_retry_after.strip()) + except (AttributeError, TypeError, ValueError): + try: + retry_at = parsedate_to_datetime(raw_retry_after) + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=timezone.utc) + retry_after = (retry_at - datetime.now(timezone.utc)).total_seconds() + except (TypeError, ValueError, OverflowError): + retry_after = None + if retry_after is not None: + retry_after = max(0.0, retry_after) + if retry_after > MAX_BACKOFF_SECONDS: + return None + return float(retry_after) + return min(backoff_factor * (2**attempt), MAX_BACKOFF_SECONDS) + + +class SafeTargetGuard: + """Validate targets and return only public IPs for connection pinning.""" + + def __init__(self, resolver: Resolver | None = None) -> None: + self._resolver = resolver or _system_resolver + + def resolve_url(self, url: str) -> ResolvedTarget: + try: + parsed = urlsplit(url) + port = parsed.port + except ValueError as exc: + raise UnsafeTarget(f"invalid URL: {exc}") from exc + if parsed.scheme.lower() not in {"http", "https"}: + raise UnsafeTarget("only HTTP and HTTPS URLs are allowed") + if parsed.username or parsed.password: + raise UnsafeTarget("URLs with credentials are not allowed") + if not parsed.hostname: + raise UnsafeTarget("URL must include a hostname") + try: + host = canonical_hostname(parsed.hostname) + except UnicodeError as exc: + raise UnsafeTarget("hostname is not valid IDNA") from exc + if host == "localhost" or host.endswith(".localhost"): + raise UnsafeTarget("localhost targets are blocked") + if host in KNOWN_METADATA_HOSTS or host.endswith(".metadata.google.internal"): + raise UnsafeTarget("cloud metadata targets are blocked") + if port == 0: + raise UnsafeTarget("port must be between 1 and 65535") + port = port or (443 if parsed.scheme.lower() == "https" else 80) + + try: + literal = ipaddress.ip_address(host) + except ValueError: + try: + raw_addresses = tuple(self._resolver(host, port)) + except OSError as exc: + raise requests.ConnectionError(f"DNS lookup failed for {host}: {exc}") from exc + if not raw_addresses: + raise requests.ConnectionError(f"DNS lookup returned no addresses for {host}") + else: + raw_addresses = (str(literal),) + + addresses: set[str] = set() + for raw_address in raw_addresses: + try: + address = ipaddress.ip_address(raw_address.split("%", 1)[0]) + except ValueError as exc: + raise UnsafeTarget(f"DNS returned an invalid address: {raw_address}") from exc + if _is_unsafe_ip(address): + raise UnsafeTarget(f"non-public target is blocked: {address}") + addresses.add(address.compressed) + return ResolvedTarget(host, port, tuple(sorted(addresses))) + + +class PinnedDNSHTTPAdapter(HTTPAdapter): + """Pin each request hop to a validated IP to prevent DNS rebinding.""" + + def __init__(self, guard: SafeTargetGuard, *args: Any, **kwargs: Any) -> None: + self.guard = guard + super().__init__(*args, **kwargs) + + def get_connection_with_tls_context( + self, + request: requests.PreparedRequest, + verify: bool | str, + proxies: Mapping[str, str] | None = None, + cert: Any = None, + ) -> Any: + if proxies: + raise requests.ProxyError("proxies are disabled for safe link checks") + target = self.guard.resolve_url(request.url or "") + host_params, pool_kwargs = self.build_connection_pool_key_attributes( + request, verify, cert + ) + scheme = str(host_params["scheme"]).lower() + original_host = target.host + display_host = f"[{original_host}]" if ":" in original_host else original_host + default_port = (scheme == "https" and target.port == 443) or ( + scheme == "http" and target.port == 80 + ) + request.headers["Host"] = ( + display_host if default_port else f"{display_host}:{target.port}" + ) + host_params["host"] = target.addresses[0] + host_params["port"] = target.port + if scheme == "https": + pool_kwargs["assert_hostname"] = original_host + pool_kwargs["server_hostname"] = original_host + return self.poolmanager.connection_from_host( + **host_params, pool_kwargs=pool_kwargs + ) + + +def build_session(guard: SafeTargetGuard) -> requests.Session: + retry = Retry( + total=0, + connect=0, + read=0, + redirect=0, + allowed_methods=frozenset({"HEAD", "GET"}), + status=0, + status_forcelist=frozenset(), + backoff_factor=0, + backoff_max=MAX_BACKOFF_SECONDS, + respect_retry_after_header=False, + raise_on_status=False, + ) + adapter = PinnedDNSHTTPAdapter( + guard, max_retries=retry, pool_connections=4, pool_maxsize=4 + ) + session = requests.Session() + session.trust_env = False + session.headers.update( + { + "User-Agent": ( + "FlyPythonCatalogLinkChecker/1.0 " + "(+https://github.com/flypythoncom/python)" + ), + "Accept": "text/html,application/xhtml+xml,application/json;q=0.8,*/*;q=0.5", + } + ) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + +class ThreadLocalSessions: + def __init__(self, factory: Callable[[], Any]) -> None: + self._factory = factory + self._local = threading.local() + self._lock = threading.Lock() + self._sessions: list[Any] = [] + + def get(self) -> Any: + session = getattr(self._local, "session", None) + if session is None: + session = self._factory() + self._local.session = session + with self._lock: + self._sessions.append(session) + return session + + def close_all(self) -> None: + with self._lock: + sessions, self._sessions = self._sessions, [] + for session in sessions: + close = getattr(session, "close", None) + if callable(close): + close() + + +class HostRateLimiter: + def __init__(self, min_interval: float) -> None: + self.min_interval = min_interval + self._master_lock = threading.Lock() + self._locks: dict[str, threading.Lock] = {} + self._last_request: dict[str, float] = {} + + def wait(self, url: str) -> None: + if self.min_interval <= 0: + return + host = canonical_hostname(urlsplit(url).hostname or "") + with self._master_lock: + lock = self._locks.setdefault(host, threading.Lock()) + with lock: + elapsed = time.monotonic() - self._last_request.get(host, 0.0) + delay = self.min_interval - elapsed + if delay > 0: + time.sleep(delay) + self._last_request[host] = time.monotonic() + + +@dataclass(frozen=True) +class CatalogLink: + id: str + path: str + title: str + url: str + + +@dataclass +class LinkResult: + id: str + path: str + title: str + url: str + status: str + status_code: int | None = None + method: str | None = None + final_url: str | None = None + history: list[dict[str, Any]] = field(default_factory=list) + error: str | None = None + + +def classify_status(status_code: int, *, redirected: bool) -> str: + if 200 <= status_code < 300: + return "redirect" if redirected else "working" + if 300 <= status_code < 400: + return "redirect" + if status_code in REVIEW_STATUS_CODES or status_code >= 500: + return "review" + if 400 <= status_code < 500: + return "broken" + return "review" + + +class LinkChecker: + def __init__( + self, + *, + timeout: float = 10.0, + workers: int = 8, + retries: int = 2, + backoff_factor: float = 0.5, + min_interval: float = 0.2, + guard: SafeTargetGuard | None = None, + session_factory: Callable[[], Any] | None = None, + ) -> None: + self.timeout = timeout + self.workers = workers + self.retries = retries + self.backoff_factor = backoff_factor + self.guard = guard or SafeTargetGuard() + factory = session_factory or (lambda: build_session(self.guard)) + self.sessions = ThreadLocalSessions(factory) + self.rate_limiter = HostRateLimiter(min_interval) + + def _request( + self, session: Any, method: str, url: str + ) -> tuple[Any, list[dict[str, Any]]]: + current_url = url + history: list[dict[str, Any]] = [] + attempt = 0 + while True: + get_adapter = getattr(session, "get_adapter", None) + adapter = get_adapter(current_url) if callable(get_adapter) else None + if not isinstance(adapter, PinnedDNSHTTPAdapter): + self.guard.resolve_url(current_url) + self.rate_limiter.wait(current_url) + kwargs: dict[str, Any] = { + "timeout": self.timeout, + "allow_redirects": False, + "stream": True, + } + try: + response = ( + session.get(current_url, **kwargs) + if method == "GET" + else session.head(current_url, **kwargs) + ) + except requests.RequestException as exc: + retryable = _is_timeout_error(exc) or isinstance( + exc, requests.ConnectionError + ) + terminal = isinstance( + exc, + ( + requests.exceptions.InvalidURL, + requests.exceptions.ProxyError, + requests.exceptions.SSLError, + ), + ) + if terminal or not retryable or attempt >= self.retries: + raise + delay = min( + self.backoff_factor * (2**attempt), MAX_BACKOFF_SECONDS + ) + attempt += 1 + if delay > 0: + time.sleep(delay) + continue + if ( + response.status_code in RETRY_STATUS_CODES + and attempt < self.retries + ): + delay = _retry_delay( + response, + attempt, + self.backoff_factor, + ) + if delay is None: + return response, history + response.close() + attempt += 1 + if delay > 0: + time.sleep(delay) + continue + + location = response.headers.get("Location") + if not 300 <= response.status_code < 400: + return response, history + if response.status_code not in REDIRECT_STATUS_CODES: + response.close() + raise RedirectProtocolError( + f"unsupported redirect status {response.status_code}" + ) + if not location: + if method == "HEAD": + return response, history + response.close() + raise RedirectProtocolError( + f"redirect status {response.status_code} has no Location header" + ) + + try: + if len(history) >= MAX_REDIRECTS: + raise requests.TooManyRedirects( + f"more than {MAX_REDIRECTS} redirects" + ) + next_url = urljoin(current_url, location) + if ( + urlsplit(current_url).scheme.lower() == "https" + and urlsplit(next_url).scheme.lower() == "http" + ): + raise UnsafeTarget("HTTPS redirects may not downgrade to HTTP") + history.append( + { + "status_code": response.status_code, + "url": current_url, + "location": location, + } + ) + finally: + response.close() + current_url = next_url + attempt = 0 + + def check_one(self, link: CatalogLink) -> LinkResult: + try: + session = self.sessions.get() + head, head_history = self._request(session, "HEAD", link.url) + try: + if ( + head.status_code < 300 + or head.status_code in REVIEW_STATUS_CODES + or head.status_code >= 500 + ): + return LinkResult( + link.id, + link.path, + link.title, + link.url, + classify_status( + head.status_code, redirected=bool(head_history) + ), + status_code=head.status_code, + method="HEAD", + final_url=head.url, + history=head_history, + ) + finally: + head.close() + + response, get_history = self._request(session, "GET", link.url) + try: + return LinkResult( + link.id, + link.path, + link.title, + link.url, + classify_status( + response.status_code, redirected=bool(get_history) + ), + status_code=response.status_code, + method="GET", + final_url=response.url, + history=get_history, + ) + finally: + response.close() + except UnsafeTarget as exc: + return LinkResult( + link.id, link.path, link.title, link.url, "blocked", error=str(exc) + ) + except requests.RequestException as exc: + if _is_timeout_error(exc): + return LinkResult( + link.id, + link.path, + link.title, + link.url, + "review", + error=f"timeout: {exc}", + ) + return LinkResult( + link.id, + link.path, + link.title, + link.url, + "error", + error=f"request failed: {exc}", + ) + except Exception as exc: # Surface checker defects without aborting other workers. + return LinkResult( + link.id, + link.path, + link.title, + link.url, + "error", + error=f"unexpected checker error: {exc}", + ) + + def check_all(self, links: Iterable[CatalogLink]) -> list[LinkResult]: + link_list = list(links) + results: list[LinkResult] = [] + try: + with ThreadPoolExecutor(max_workers=self.workers) as executor: + futures = { + executor.submit(self.check_one, link): link for link in link_list + } + for future in as_completed(futures): + results.append(future.result()) + finally: + self.sessions.close_all() + return sorted(results, key=lambda result: (result.path, result.id, result.url)) + + +def select_links( + data: Mapping[str, Any], *, mode: str, base_url: str +) -> list[CatalogLink]: + base_host = canonical_hostname(urlsplit(base_url).hostname or "") + links: list[CatalogLink] = [] + for resource in catalog_resources(data): + raw_url = str(resource.get("url", "")) + absolute_url = urljoin(base_url, raw_url) + host = canonical_hostname(urlsplit(absolute_url).hostname or "") + is_internal = host == base_host + if mode == "internal" and not is_internal: + continue + if mode == "external" and is_internal: + continue + links.append( + CatalogLink( + id=str(resource.get("id", "")), + path=str(resource.get("path", "")), + title=str(resource.get("title", "")), + url=absolute_url, + ) + ) + return links + + +def build_report( + *, catalog: Path, mode: str, results: list[LinkResult] +) -> dict[str, Any]: + counts = { + status: sum(item.status == status for item in results) + for status in REPORT_STATUSES + } + counts["total"] = len(results) + return { + "schema_version": 1, + "generated_at": datetime.now(timezone.utc).isoformat(), + "catalog": str(catalog), + "mode": mode, + "counts": counts, + "results": [asdict(result) for result in results], + } + + +def exit_code_for_report(report: Mapping[str, Any]) -> int: + if not isinstance(report, Mapping): + return 2 + counts = report.get("counts", {}) + if not isinstance(counts, Mapping): + return 2 + expected_keys = {*REPORT_STATUSES, "total"} + if set(counts) != expected_keys: + return 2 + values = [counts[status] for status in REPORT_STATUSES] + total = counts["total"] + if ( + not isinstance(total, int) + or isinstance(total, bool) + or total <= 0 + or any( + not isinstance(value, int) or isinstance(value, bool) or value < 0 + for value in values + ) + or sum(values) != total + ): + return 2 + if any(counts[status] for status in ACTIONABLE_STATUSES): + return 1 + return 0 + + +def _positive_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed) or parsed <= 0: + raise argparse.ArgumentTypeError("must be a finite number greater than zero") + return parsed + + +def _non_negative_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed) or parsed < 0: + raise argparse.ArgumentTypeError("must be a finite number zero or greater") + return parsed + + +def _workers(value: str) -> int: + parsed = int(value) + if not 1 <= parsed <= 32: + raise argparse.ArgumentTypeError("must be between 1 and 32") + return parsed + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--mode", choices=("internal", "external", "all"), default="all") + mode.add_argument("--internal", action="store_const", const="internal", dest="mode") + mode.add_argument("--external", action="store_const", const="external", dest="mode") + mode.add_argument("--all", action="store_const", const="all", dest="mode") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL) + parser.add_argument( + "--output", default=str(DEFAULT_OUTPUT), help="JSON path, or '-' for stdout" + ) + parser.add_argument("--timeout", type=_positive_float, default=10.0) + parser.add_argument("--workers", type=_workers, default=8) + parser.add_argument("--retries", type=int, default=2) + parser.add_argument("--backoff", type=_non_negative_float, default=0.5) + parser.add_argument("--min-interval", type=_non_negative_float, default=0.2) + return parser + + +def run(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.retries < 0: + print("--retries must be zero or greater", file=sys.stderr) + return 2 + try: + data = load_catalog(args.catalog) + except CatalogLoadError as exc: + print(str(exc), file=sys.stderr) + return 2 + issues = validate_catalog(data) + if issues: + print("catalog validation failed before link checks", file=sys.stderr) + for issue in issues: + print(f"{issue.location}: {issue.code}: {issue.message}", file=sys.stderr) + return 2 + + links = select_links(data, mode=args.mode, base_url=args.base_url) + checker = LinkChecker( + timeout=args.timeout, + workers=args.workers, + retries=args.retries, + backoff_factor=args.backoff, + min_interval=args.min_interval, + ) + results = checker.check_all(links) + report = build_report(catalog=args.catalog, mode=args.mode, results=results) + payload = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + if args.output == "-": + sys.stdout.write(payload) + else: + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(payload, encoding="utf-8") + print(f"link report: {output}", file=sys.stderr) + counts = report["counts"] + print( + "links: " + ", ".join(f"{key}={value}" for key, value in counts.items()), + file=sys.stderr, + ) + return exit_code_for_report(report) + + +def main() -> None: + raise SystemExit(run()) + + +if __name__ == "__main__": + main() diff --git a/tools/validate_catalog.py b/tools/validate_catalog.py new file mode 100644 index 0000000..26e3dcb --- /dev/null +++ b/tools/validate_catalog.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Validate the canonical FlyPython resource catalog.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +try: + from tools.catalog import CatalogLoadError, load_catalog, validate_catalog +except ModuleNotFoundError: # Direct ``python tools/validate_catalog.py`` execution. + from catalog import CatalogLoadError, load_catalog, validate_catalog + + +ROOT_DIR = Path(__file__).resolve().parent.parent + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--catalog", type=Path, default=ROOT_DIR / "_data" / "resources.yml" + ) + parser.add_argument("--output", type=Path, help="optional JSON report path") + parser.add_argument("--max-review-age-days", type=int, default=366) + return parser + + +def run(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.max_review_age_days < 1: + print("--max-review-age-days must be positive", file=sys.stderr) + return 2 + try: + data = load_catalog(args.catalog) + except CatalogLoadError as exc: + print(str(exc), file=sys.stderr) + return 2 + issues = validate_catalog(data, max_review_age_days=args.max_review_age_days) + report = { + "catalog": str(args.catalog), + "valid": not issues, + "issue_count": len(issues), + "issues": [issue.as_dict() for issue in issues], + } + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + if issues: + for issue in issues: + print( + f"{issue.location}: {issue.code}: {issue.message}", file=sys.stderr + ) + return 1 + print(f"catalog valid: {len(data.get('resources', []))} resources") + return 0 + + +def main() -> None: + raise SystemExit(run()) + + +if __name__ == "__main__": + main() diff --git a/unp/1/download.py b/unp/1/download.py deleted file mode 100644 index 253caf1..0000000 --- a/unp/1/download.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import urllib,sys - -f=urllib.urlopen(sys.argv[1]) - -while True: - buf = f.read(2048) - if not len(buf): - break - sys.stdout.write(buf) diff --git a/unp/1/gopherclient.py b/unp/1/gopherclient.py deleted file mode 100644 index d5c01ce..0000000 --- a/unp/1/gopherclient.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -# --coding: utf-8 -- -#简单的Gopher Client - -import socket,sys - -port = 70 #默认端口 70 -host = sys.argv[1] -filename = sys.argv[2] - -s= socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.connect((host,port)) - -s.sendall(filename + "\r\n") - -while 1: - buf = s.recv(2048) - if not len(buf): - break - sys.stdout.write(buf) - - diff --git a/unp/1/gopherclient2.py b/unp/1/gopherclient2.py deleted file mode 100644 index 5b1b954..0000000 --- a/unp/1/gopherclient2.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -# --coding: utf-8 -- -#简单的Gopher Client - -import socket,sys - -port = 70 #默认端口 70 -host = sys.argv[1] -filename = sys.argv[2] - -s= socket.socket(socket.AF_INET,socket.SOCK_STREAM) - -try: - s.connect((host,port)) -except socket.gaierror,e: - print "Error connecting to server:%s" % e - sys.exit(1) - -s.sendall(filename + "\r\n") - -while 1: - buf = s.recv(2048) - if not len(buf): - break - sys.stdout.write(buf) - - diff --git a/unp/1/gopherclient3.py b/unp/1/gopherclient3.py deleted file mode 100644 index 4bd6831..0000000 --- a/unp/1/gopherclient3.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -# --coding: utf-8 -- -#简单的Gopher Client - -import socket,sys - -port = 70 #默认端口 70 -host = sys.argv[1] -filename = sys.argv[2] - -s= socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.connect((host,port)) - -fd = s.makefile('rw',0) - -fd.write(filename + "\r\n") - -for line in fd.readline(): - sys.stdout.write(line) - - - - - - - - diff --git a/unp/1/server.py b/unp/1/server.py deleted file mode 100644 index ca441c7..0000000 --- a/unp/1/server.py +++ /dev/null @@ -1,25 +0,0 @@ -#! /usr/bin env python - -import socket - -host='' -porr=51423 - -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) -s.bind((host,porr)) -s.listen(1) - -print "server is runing on port %d" % port - -while 1: - clientsock,clientaddr=s.accept() - clientfile=clientsock.makefile('rw',0) - clientfile.write("welcome," + str(clientaddr) + "\n") - clientfile.write("please enter a string:") - line = clientfile.readline().strip() - clientfile.write("you enter %d char" % len(line) ) - clientfile.close() - clientsock.close() - - diff --git a/unp/2/connect.py b/unp/2/connect.py deleted file mode 100644 index 2938a35..0000000 --- a/unp/2/connect.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin env python - -import socket - -print "create socket" -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -print "done" - - -print "connecting to the host" -s.connect(('localhost',8888)) -print "done" diff --git a/unp/2/connect1.py b/unp/2/connect1.py deleted file mode 100644 index f1efdd4..0000000 --- a/unp/2/connect1.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin env python - -import socket - -print "create socket" -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -print "done" - -print "look up port number" -port=socket.getservbyname('http','tcp') -print "done" - - -print "connecting to the host on port %d" % port -s.connect(("www.baidu.com",port)) -print "done" diff --git a/unp/2/connect3.py b/unp/2/connect3.py deleted file mode 100644 index 559e5fa..0000000 --- a/unp/2/connect3.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin env python - -import socket - -print "create socket" -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -print "done" - -print "look up port number" -port=socket.getservbyname('http','tcp') -print "done" - - -print "connecting to the host on port %d" % port -s.connect(("www.baidu.com",port)) -print "done" - -print "connected from",s.getsockname() -print "connected to",s.getpeername() diff --git a/unp/2/socketerrors.py b/unp/2/socketerrors.py deleted file mode 100644 index f7dc995..0000000 --- a/unp/2/socketerrors.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin env python - -import socket,sys - -host=sys.argv[1] -textport=sys.argv[2] -filename=sys.argv[3] - -try: - s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -except socket.error,e: - print "strange erroe create socket %s" % e - sys.exit(1) - - -try: - port = int(textport) -except ValueError: - try: - port=socket.getservbyname(textport,'tcp') - except socket.error,e: - print "could find you port %s" % e - sys.exit(1) - diff --git a/unp/2/udp.py b/unp/2/udp.py deleted file mode 100644 index 7c950e5..0000000 --- a/unp/2/udp.py +++ /dev/null @@ -1,31 +0,0 @@ -#! /usr/bin env python - -import socket,sys - -host=sys.argv[1] -textport=sys.argv[2] - -s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) - -try: - port=int(textport) -except ValueError: - port=socket.getservbyname(textport,'udp') - - -s.connect((host,port)) - -print "enter data to tranmit:" - -data=sys.stdin.readline().strip() -s.sendall(data) - -print "looking for replies;" - -while 1: - buf=s.recv(2048) - if not len(buf): - break - sys.stdout.write(buf) - - diff --git a/unp/3/basicserver.py b/unp/3/basicserver.py deleted file mode 100644 index 1977ee0..0000000 --- a/unp/3/basicserver.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin env python - -import socket - -host='' -port=8888 - -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) -s.bind((host,port)) -print "waiting for connect..." -s.listen(1) - -while 1: - clientsock,clientaddr=s.accept() - print "Go connection from ,", clientsock.getpeername() - clientsock.close() - diff --git a/unp/3/echoclient.py b/unp/3/echoclient.py deleted file mode 100644 index 7b5e002..0000000 --- a/unp/3/echoclient.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin env python - -import sys,socket - -host='localhost' -port=8888 - -data="x"*1024 - -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.connect((host,port)) - -bytewritten=0 -while bytewritten < len(data): - startpos=bytewritten - endpos=min(bytewritten + 1024,len(data)) - bytewritten += s.send(data[startpos:endpos]) - sys.stdout.write("write %d bytes\r" % bytewritten) - sys.stdout.flush() - -s.shutdown(1) - -print "All data sent" -while 1: - buf=s.recv(1024) - if not len(buf): - break - sys.stdout.write(buf) - - diff --git a/unp/3/echoserver.py b/unp/3/echoserver.py deleted file mode 100644 index 4e2a6ee..0000000 --- a/unp/3/echoserver.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin env python -#!--coding: utf-8 -- - -import socket,traceback - -host='' -port=8888 - -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) -s.bind((host,port)) -s.listen(1) - -while 1: - try: - clientsock,clientaddr=s.accept() - except KeyboardInterrupt: - raise - except: - traceback.print_exc() - continue - - try: - print "Got connection from " ,clientsock.getpeername() - while 1: - data= clientsock.recv(4096) - if not len(data): - break - clientsock.sendall(data) - except (KeyboardInterrupt,SystemExit): - raise - except: - traceback.print_exc() - try: - clientsock.close() - except KeyboardInterrupt: - raise - except: - traceback.print_exc() - - - - - diff --git a/unp/3/socketop.py b/unp/3/socketop.py deleted file mode 100644 index 073e6dd..0000000 --- a/unp/3/socketop.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin env python - -import socket - -solist = [x for x in dir(socket) if x.startswith('SO_') ] -solist.sort() - -for x in solist: - print x diff --git a/unp/3/syslogsample.py b/unp/3/syslogsample.py deleted file mode 100644 index 6faf4a2..0000000 --- a/unp/3/syslogsample.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin env python -#!--coding: utf-8-- - -import syslog,StringIO,sys,os,traceback - -def logexception(includetraceback=0): - exctype,exception,exctraceback=sys.exc_info() - excclass=str(exception.__class__) - message=str(exception) - - if not includetraceback: - syslog.syslog(syslog.LOG_ERR,"%s: %s" % (excclass,message)) - else: - excfd=StringIO.StringIO() - traceback.print_exception(exctype,exception,exctraceback,None,excfd) - - for line in excfd.getvalue().split("\n"): - syslog.syslog(syslog.LOG_ERR,line) - -def initsyslog(): - syslog.openlog("%s[%d]" % (os.path.basename(sys.argv[0]),os.getpid()),0, - syslog.LOG_DAEMON) - syslog.syslog("started.") - - -initsyslog() -try: - raise RuntimeError, "Exception 1" -except: - logexception(0) - -try: - raise RuntimeError, "Exception 2" -except: - logexception(1) - -syslog.syslog("I am a terimiting.") - - - diff --git a/unp/3/udpechoserver.py b/unp/3/udpechoserver.py deleted file mode 100644 index dc6cdcb..0000000 --- a/unp/3/udpechoserver.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin env python - -import socket,traceback - -host='' -port=8888 - -s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) -s.bind((host,port)) - -while 1: - try: - message,address=s.recvfrom(8192) - print "Go data from",address - s.sendall(message,address) - except (keyboardInterrupt,SystemExit): - raise - except: - traceback.print_exc() - - diff --git a/unp/3/xinetdserver.py b/unp/3/xinetdserver.py deleted file mode 100644 index 492b0cb..0000000 --- a/unp/3/xinetdserver.py +++ /dev/null @@ -1,11 +0,0 @@ -#/usr/bin env python - -import sys -print "welcome...." -print "please enter a string:" - -sys.stdout.flush() -line=sys.stdin.readline().strip() - - -print "you enter the %s is %d count" % (line,len(line)) diff --git a/unp/4/dns.py b/unp/4/dns.py deleted file mode 100644 index 5923b53..0000000 --- a/unp/4/dns.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin env python - -import sys,DNS - -query=sys.argv[1] -DNS.DiscoverNameServers() - -reqobj=DNS.Request() - -answerobj=reqobj.req(name=query,qtype=DNS.Type.ANY) -if not len(answerobj.answers): - print "not find" -for i in answerobj.answers: - print "%-5s %s " % (i['typename'],i['data']) - diff --git a/unp/4/dns.pyc b/unp/4/dns.pyc deleted file mode 100644 index a54fc76..0000000 Binary files a/unp/4/dns.pyc and /dev/null differ diff --git a/unp/4/environment.py b/unp/4/environment.py deleted file mode 100644 index e29251f..0000000 --- a/unp/4/environment.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin env pythhon - -import sys,socket - -def getipaddrs(hostname): - s=socket.getaddrinfo(hostname,None,0,socket.SOCK_STREAM) - return [x[4][0] for x in s] - -hostname=socket.gethostname() -print "Host name:",hostname - -print "full-name:",socket.getfqdn(hostname) -try: - print "IP address:",", ".join(getipaddrs(hostname)) -except socket.gaierror,e: - print "can not get ip address",e - diff --git a/unp/4/getaddrinfo.py b/unp/4/getaddrinfo.py deleted file mode 100644 index 5d01ae6..0000000 --- a/unp/4/getaddrinfo.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin env python - -import socket,sys - -s=socket.getaddrinfo(sys.argv[1],None,0,socket.SOCK_STREAM) - -counter=0 -for i in s: - print "%d %s" % (counter,s[counter][4]) - counter += 1 - - diff --git a/unp/4/gethostbyaddr.py b/unp/4/gethostbyaddr.py deleted file mode 100644 index 4eb61c5..0000000 --- a/unp/4/gethostbyaddr.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin env python - -import sys,socket - -try: - - s=socket.gethostbyaddr(sys.argv[1]) - - print "hostname:" - print " "+ s[0] - - print "\nAddress:" - for i in s[2]: - print " " + i - -except socket.herror,e: - print "can not look up name:",e - - diff --git a/unp/5/bcastclinet.py b/unp/5/bcastclinet.py deleted file mode 100644 index 8ed2d3a..0000000 --- a/unp/5/bcastclinet.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin env python - -import socket,traceback - -host='' -port=8888 - - -s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) -s.setsockopt(socket.SOL_SOCKET,socket.SO_BROADCAST,1) -s.bind((host,port)) - -while 1: - try: - message,address=s.recvfrom(8192) - print "Got data from",addrrss - s.sendto("I am here",address) - except (KeyboardInterrupt,SystemExit): - raise - except: - traceback.print_exc() - - - diff --git a/unp/5/bcastserver.py b/unp/5/bcastserver.py deleted file mode 100644 index 0332651..0000000 --- a/unp/5/bcastserver.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin env python - -import sys,socket - -dest=('',8888) - -s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_BROADCAST,1) -s.sendto("hello",dest) - -print "look for replies ...." -while 1: - (buf,address)=s.recvfrom(2048) - if not len(buf): - break - print "Recived from %s: %s" % (address,buf) diff --git a/unp/5/delayserver.py b/unp/5/delayserver.py deleted file mode 100644 index 388d6f9..0000000 --- a/unp/5/delayserver.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin env python - -import socket,traceback,time - -host='' -port=8888 - -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) -s.bind((host,port)) - -s.listen(1) - - -while 1: - try: - clientsock,clientaddr= s.accept() - except KeyboardInterrupt: - raise - except: - traceback.print_exc() - continue - - try: - print "get connect from",clientsock.getpeername() - while 1: - try: - clientsock.sendall(time.asctime() + "\n") - except: - break - time.sleep(5) - except (KeyboardInterrupt,SystemExit): - raise - except: - traceback.print_exc() -try: - clientsock.close() -except KeyboardInterrupt: - raise -except: - traceback.print_exc() - -try: - clientsock.close() -except KeyboardInterrupt: - raise -except: - traceback.print_exc() - - - - diff --git a/unp/5/nbo.py b/unp/5/nbo.py deleted file mode 100644 index f82760b..0000000 --- a/unp/5/nbo.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin env python - -import struct,sys - -def htons(num): - return struct.pack('!H',num) - -def htonl(num): - return struct.pack('!I',num) - -def ntohs(data): - return struct.unpack('!H',data)[0] - -def ntohl(data): - return struct.unpack("!I",data)[0] - -def sendstring(data): - return htonl(len(data)) + data - -print "Enter a string:" -str=sys.stdin.readline().rstrip() - -print repr(sendstring(str)) diff --git a/unp/5/pollclient.py b/unp/5/pollclient.py deleted file mode 100644 index 61c32b0..0000000 --- a/unp/5/pollclient.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin env python - -import socket,sys,select -port=8888 -host='localhost' - -spinsize=10 -spinpos=0 -spindir=1 - -def spin(): - global spinsize,spinpos,spindir - spinstr='.' * spinpos + '|'+'.'(spinsize-spinpos-1) - sys.stdout.write('r'+ spinstr + ' ') - sys.stdout.flush() - - spinpos += spindir - if spinpos < 0: - spindir=1 - spinpos=1 - elif spinpos >= spinsize: - spinpos -= 2 - spindir = -1 - -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.connect((host,port)) - -p=select.poll() -p.register(s.fileno(),select.POLLIN | select.POLLERR | select.POLLHUP) - -while 1: - results=p.poll(50) - if len(results): - if results[0][1] == select.POLLIN: - data = s.recv(4096) - if not len(data): - print "Remote end closed connect" - break - sys.stdout.write("\rReceived:" + data) - sys.stdout.flush() - else: - print "\rproblem occurred ; exiting" - sys.exit(0) - -spin() - - diff --git a/unp/5/selectclient.py b/unp/5/selectclient.py deleted file mode 100644 index 03cc94b..0000000 --- a/unp/5/selectclient.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin env python - -import socket,sys,select -port=8888 -host='localhost' - -spinsize=10 -spinpos=0 -spindir=1 - -def spin(): - global spinsize,spinpos,spindir - spinstr='.' * spinpos + '|' +'.'*(spinsize-spinpos-1) - sys.stdout.write('r'+ spinstr + ' ') - sys.stdout.flush() - - spinpos += spindir - if spinpos < 0: - spindir=1 - spinpos=1 - elif spinpos >= spinsize: - spinpos -= 2 - spindir = -1 - -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.connect((host,port)) - -while 1: - infds,outfds,errfds=select.select([s],[],[s],0.05) - if len(infds): - data=s.recv(4096) - if not len(data): - print "\rRemote end closed connection; exiting" - break - - sys.stdout.write("\rRecived: " + data) - sys.stdout.flush() - - if len(errfds): - print "\r problem occurred; exiting" - sys.exit(0) - -spin() - - - - - - - - - - - - - - diff --git a/unp/5/timeoutserver.py b/unp/5/timeoutserver.py deleted file mode 100644 index 229476d..0000000 --- a/unp/5/timeoutserver.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin env python - -import socket,traceback - -host='' -port=8888 - -s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) -s.bind((host,port)) - -s.listen(1) - - -while 1: - try: - clientsock,clientaddr= s.accept() - except KeyboardInterrupt: - raise - except: - traceback.print_exc() - continue - clientsock.settimeout(5) - - try: - print "get connect from",clientsock.getpeername() - while 1: - data=clientsock.recv(4096) - if not len(data): - break - clientsock.sendall(data) - except (KeyboardInterrupt,SystemExit): - raise - except socket.timeout: - pass - except: - traceback.print_exc() - - try: - clientsock.close() - except KeyboardInterrupt: - raise - except: - traceback.print_exc() - - - - diff --git a/unp/6/dump_info.py b/unp/6/dump_info.py deleted file mode 100644 index d351719..0000000 --- a/unp/6/dump_info.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin env python - -import sys,urllib2 - -req=urllib2.Request(sys.argv[1]) -fd=urllib2.urlopen(req) - -print "Retrieved",fd.geturl() -info=fd.info() - -for key,value in info.items(): - print "%s = %s " % ( key,value) diff --git a/unp/6/dump_page.py b/unp/6/dump_page.py deleted file mode 100644 index b550f63..0000000 --- a/unp/6/dump_page.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin env python - -import sys,urllib2 - -req=urllib2.Request(sys.argv[1]) -fd=urllib2.urlopen(req) - -while 1: - data=fd.read(1024) - if not len(data): - break - - sys.stdout.write(data) diff --git a/zh-cn.md b/zh-cn.md new file mode 100644 index 0000000..4a36288 --- /dev/null +++ b/zh-cn.md @@ -0,0 +1,41 @@ +--- +layout: default +title: 面向 AI 时代的 Python 资源目录 +description: 经过审核的 Python 一手资源目录，覆盖基础、Web API、自动化与 AI Agent。 +lang: zh-CN +permalink: /zh/ +image: + path: /assets/images/og-image.png + width: 1200 + height: 630 + alt: FlyPython Python 资源目录 +--- +{% assign catalog_resources = site.data.resources.resources | where: "status", "active" %} + +# 从值得信任的一手来源学习 Python。 + +FlyPython 收录官方文档、正式标准和项目的一手资源。通过四条路径找到下一份可靠资料， +再回到主学习站，用完整项目和清晰上下文把知识真正用起来。 + +
+ 从学习路线开始 + 浏览主站精选资源 + 在 GitHub 参与维护 +
+ +

+ {{ catalog_resources.size }} 条有效资源 + + 目录审核于 {{ site.data.resources.catalog.reviewed_on | escape }} + + 一手来源优先 +

+ +{% include catalog-grid.html resources=catalog_resources %} + +## 这个目录与 FlyPython 主站的关系 + +本仓库公开维护覆盖面更广的社区资源地图；[FlyPython 学习站](https://flypython.com/) +从中挑选更小的一组来源，组织成实用学习路径、经过测试的示例和清晰的下一步。 +如果你想先完成一个项目再选择 Agent 框架，可以从 +[无需 API Key 的 Python Agent 指南](https://flypython.com/learn/python-ai-agent-roadmap)开始。 diff --git "a/\345\215\232\345\256\242/Python yield \347\224\250\346\263\225.pdf" "b/\345\215\232\345\256\242/Python yield \347\224\250\346\263\225.pdf" deleted file mode 100644 index 2ab8971..0000000 Binary files "a/\345\215\232\345\256\242/Python yield \347\224\250\346\263\225.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/Python \344\273\243\347\240\201\346\200\247\350\203\275\344\274\230\345\214\226\346\212\200\345\267\247.pdf" "b/\345\215\232\345\256\242/Python \344\273\243\347\240\201\346\200\247\350\203\275\344\274\230\345\214\226\346\212\200\345\267\247.pdf" deleted file mode 100644 index 3bd60a6..0000000 Binary files "a/\345\215\232\345\256\242/Python \344\273\243\347\240\201\346\200\247\350\203\275\344\274\230\345\214\226\346\212\200\345\267\247.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/Python \345\274\200\345\217\221\350\200\205\345\272\224\350\257\245\347\237\245\351\201\223\347\232\204 7 \344\270\252\345\274\200\345\217\221\345\272\223.pdf" "b/\345\215\232\345\256\242/Python \345\274\200\345\217\221\350\200\205\345\272\224\350\257\245\347\237\245\351\201\223\347\232\204 7 \344\270\252\345\274\200\345\217\221\345\272\223.pdf" deleted file mode 100644 index e77b65a..0000000 Binary files "a/\345\215\232\345\256\242/Python \345\274\200\345\217\221\350\200\205\345\272\224\350\257\245\347\237\245\351\201\223\347\232\204 7 \344\270\252\345\274\200\345\217\221\345\272\223.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/python ftp\347\232\204\346\226\255\347\202\271\344\270\212\344\274\240.pdf" "b/\345\215\232\345\256\242/python ftp\347\232\204\346\226\255\347\202\271\344\270\212\344\274\240.pdf" deleted file mode 100644 index 99c6cff..0000000 Binary files "a/\345\215\232\345\256\242/python ftp\347\232\204\346\226\255\347\202\271\344\270\212\344\274\240.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/python \345\207\275\346\225\260\344\277\256\351\245\260\345\231\250.pdf" "b/\345\215\232\345\256\242/python \345\207\275\346\225\260\344\277\256\351\245\260\345\231\250.pdf" deleted file mode 100644 index 9ffd80d..0000000 Binary files "a/\345\215\232\345\256\242/python \345\207\275\346\225\260\344\277\256\351\245\260\345\231\250.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/python\345\206\231\346\225\260\346\215\256\347\273\223\346\236\204.pdf" "b/\345\215\232\345\256\242/python\345\206\231\346\225\260\346\215\256\347\273\223\346\236\204.pdf" deleted file mode 100644 index e4af161..0000000 Binary files "a/\345\215\232\345\256\242/python\345\206\231\346\225\260\346\215\256\347\273\223\346\236\204.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/python\347\210\254\350\231\253\346\212\223\347\253\231\347\232\204\346\200\273\347\273\223.pdf" "b/\345\215\232\345\256\242/python\347\210\254\350\231\253\346\212\223\347\253\231\347\232\204\346\200\273\347\273\223.pdf" deleted file mode 100644 index c6345fe..0000000 Binary files "a/\345\215\232\345\256\242/python\347\210\254\350\231\253\346\212\223\347\253\231\347\232\204\346\200\273\347\273\223.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-epoll.pdf" "b/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-epoll.pdf" deleted file mode 100644 index f260cd0..0000000 Binary files "a/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-epoll.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-\345\272\225\345\261\202\347\275\221\347\273\234.pdf" "b/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-\345\272\225\345\261\202\347\275\221\347\273\234.pdf" deleted file mode 100644 index e362f1d..0000000 Binary files "a/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-\345\272\225\345\261\202\347\275\221\347\273\234.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-\346\225\260\346\215\256\345\272\223.pdf" "b/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-\346\225\260\346\215\256\345\272\223.pdf" deleted file mode 100644 index 36a3607..0000000 Binary files "a/\345\215\232\345\256\242/python\347\275\221\347\273\234\347\274\226\347\250\213-\346\225\260\346\215\256\345\272\223.pdf" and /dev/null differ diff --git "a/\345\215\232\345\256\242/python\351\235\242\350\257\225\351\242\230.pdf" "b/\345\215\232\345\256\242/python\351\235\242\350\257\225\351\242\230.pdf" deleted file mode 100644 index cc562a4..0000000 Binary files "a/\345\215\232\345\256\242/python\351\235\242\350\257\225\351\242\230.pdf" and /dev/null differ