CI/CD Python 项目

# .github/workflows/ci.yml
name: CI

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

jobs:
  quality:
    name: 质量门禁
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: 安装依赖
        run: |
          pip install --quiet ruff mypy
          pip install --quiet -e ".[dev]"

      - name: Ruff lint
        run: ruff check .

      - name: Ruff format 检查
        run: ruff format --check .

      - name: mypy 类型检查
        run: mypy src/ --ignore-missing-imports

  test:
    name: 测试 (Python ${{ matrix.python-version }})
    needs: quality
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
      fail-fast: false

    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_USER: test
          POSTGRES_PASSWORD: test
          POSTGRES_DB: testdb
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U test"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: 缓存 pip 依赖
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: pip-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            pip-${{ runner.os }}-${{ matrix.python-version }}-

      - name: 安装依赖
        run: |
          pip install --quiet -e ".[dev,test]"

      - name: 运行测试
        env:
          DATABASE_URL: postgresql://test:test@localhost:5432/testdb
        run: |
          pytest tests/ \
            --cov=src \
            --cov-report=xml \
            --cov-report=term-missing \
            -v --tb=short

      - name: 上传覆盖率
        if: matrix.python-version == '3.12'
        uses: codecov/codecov-action@v4
        with:
          file: ./coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}

  build:
    name: 构建产物
    needs: test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0  # setuptools-scm 需要完整 git 历史

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: 安装构建工具
        run: pip install --quiet build

      - name: 构建 Wheel 和 Sdist
        run: python -m build

      - name: 检查产物
        run: |
          pip install --quiet twine
          twine check dist/*

      - uses: actions/upload-artifact@v4
        with:
          name: dist
          path: dist/

  publish-pypi:
    name: 发布到 PyPI
    needs: build
    if: startsWith(github.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    environment: pypi
    permissions:
      id-token: write  # Trusted Publishing 必需

    steps:
      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: dist/

      - name: 发布到 PyPI（Trusted Publishing）
        uses: pypa/gh-action-pypi-publish@release/v1

  github-release:
    name: 创建 GitHub Release
    needs: publish-pypi
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: 生成 Changelog
        id: changelog
        run: |
          PREV_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
          if [ -n "$PREV_TAG" ]; then
            LOG=$(git log ${PREV_TAG}..HEAD --pretty=format:"- %s (%h)")
          else
            LOG=$(git log --pretty=format:"- %s (%h)")
          fi
          echo "changelog<<EOF" >> $GITHUB_OUTPUT
          echo "$LOG" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: dist/

      - name: 创建 Release
        uses: softprops/action-gh-release@v2
        with:
          body: ${{ steps.changelog.outputs.changelog }}
          files: dist/*

# .gitlab-ci.yml
stages:
  - quality
  - test
  - build
  - publish
  - deploy

variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  POSTGRES_USER: test
  POSTGRES_PASSWORD: test
  POSTGRES_DB: testdb

cache:
  key:
    files:
      - pyproject.toml
  paths:
    - .cache/pip
    - .venv/

# ── 质量门禁 ──
ruff-lint:
  stage: quality
  image: python:3.12-slim
  before_script:
    - pip install --quiet ruff
    - pip install --quiet -e ".[dev]"
  script:
    - ruff check .
    - ruff format --check .

mypy-check:
  stage: quality
  image: python:3.12-slim
  before_script:
    - pip install --quiet mypy
    - pip install --quiet -e ".[dev]"
  script:
    - mypy src/ --ignore-missing-imports

# ── 测试 ──
test:
  stage: test
  image: python:3.12-slim
  services:
    - name: postgres:16
      alias: postgres
  variables:
    DATABASE_URL: postgresql://test:test@postgres:5432/testdb
  before_script:
    - pip install --quiet -e ".[dev,test]"
  script:
    - pytest tests/ --cov=src --cov-report=xml --cov-report=term-missing -v
  coverage: '/TOTAL.*\s+(\d+%)$/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml

# ── 构建 ──
build-wheel:
  stage: build
  image: python:3.12-slim
  script:
    - pip install --quiet build
    - python -m build
  artifacts:
    paths:
      - dist/
    expire_in: 7 days

build-docker:
  stage: build
  image: docker:24
  services:
    - docker:24-dind
  variables:
    DOCKER_TLS_CERTDIR: "/certs"
  before_script:
    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
  script:
    - |
      docker build \
        --build-arg VERSION=$CI_COMMIT_TAG \
        -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA \
        -t $CI_REGISTRY_IMAGE:latest \
        .
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
    - docker push $CI_REGISTRY_IMAGE:latest
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_TAG

# ── 发布 ──
publish-pypi:
  stage: publish
  image: python:3.12-slim
  needs:
    - build-wheel
  script:
    - pip install --quiet twine
    - twine upload dist/* --username __token__ --password $PYPI_TOKEN
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/

# ── 部署 ──
deploy-staging:
  stage: deploy
  image: alpine:3.19
  before_script:
    - apk add --no-cache openssh-client
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
  script:
    - ssh -o StrictHostKeyChecking=no deploy@staging.example.com "cd /app && docker compose pull && docker compose up -d"
  environment:
    name: staging
    url: https://staging.example.com
  rules:
    - if: $CI_COMMIT_BRANCH == "main"

deploy-production:
  stage: deploy
  extends: deploy-staging
  script:
    - ssh -o StrictHostKeyChecking=no deploy@prod.example.com "cd /app && docker compose pull && docker compose up -d"
  environment:
    name: production
    url: https://app.example.com
  when: manual  # 需要手动触发
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/

# pyproject.toml
[project]
name = "mylib"
dynamic = ["version"]

[tool.setuptools_scm]
# 从 git tag 自动推导版本号
# tag v1.2.3 → version 1.2.3
# tag v1.2.3 + 3 commits → version 1.2.4.dev3+gabc1234
fallback_version = "0.0.0"

# pyproject.toml
[tool.semantic_release]
version_toml = ["pyproject.toml:project.version"]
changelog_file = "CHANGELOG.md"
build_command = "pip install build && python -m build"
dist_path = "dist/"
upload_to_pypi = false  # 由 CI 单独处理发布
major_on_zero = false
allow_zero_version = true
commit_parser = "angular"

# .github/workflows/release.yml
name: Release

on:
  push:
    branches: [main]

jobs:
  release:
    runs-on: ubuntu-latest
    concurrency: release
    permissions:
      contents: write
      id-token: write

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          token: ${{ secrets.GH_PAT }}

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: 安装 semantic-release
        run: pip install python-semantic-release

      - name: 运行 semantic-release
        run: semantic-release version

      - name: 构建产物
        run: pip install build && python -m build

      - name: 发布到 PyPI
        uses: pypa/gh-action-pypi-publish@release/v1

      - name: 创建 GitHub Release
        run: semantic-release publish

# pyproject.toml
[tool.towncrier]
package = "mylib"
directory = "changelog.d"
filename = "CHANGELOG.md"
issue_format = "[#{issue}](https://github.com/user/mylib/issues/{issue})"
type = [
    { name = "Breaking Changes",   directory = "breaking", showcontent = true },
    { name = "New Features",       directory = "feature",  showcontent = true },
    { name = "Bug Fixes",          directory = "bugfix",   showcontent = true },
    { name = "Documentation",      directory = "doc",      showcontent = true },
    { name = "Internal Changes",   directory = "internal", showcontent = false },
]

# changelog.d/ 目录结构
# 每个变更一个文件，格式: <issue_number>.<type>.md
changelog.d/
  42.feature.md    # "新增了用户注册功能"
  57.bugfix.md     # "修复了登录超时问题"
  60.breaking.md   # "移除了废弃的旧 API"

# 发布时自动合并 changelog
towncrier build --version 1.3.0 --yes
# 此命令会：
# 1. 读取 changelog.d/ 下的所有 fragment
# 2. 按类型分组，追加到 CHANGELOG.md
# 3. 删除已合并的 fragment 文件

# ── 第一阶段：构建依赖 ──
FROM python:3.12-slim AS builder

WORKDIR /build

# 先复制依赖声明，利用 Docker 缓存层
COPY pyproject.toml README.md ./
COPY src/ src/

# 安装依赖到独立目录
RUN pip install --no-cache-dir --prefix=/install .

# ── 第二阶段：运行时镜像 ──
FROM python:3.12-slim AS runtime

# 安全：非 root 用户运行
RUN groupadd -r appuser && useradd -r -g appuser appuser

WORKDIR /app

# 仅从 builder 复制已安装的依赖和代码
COPY --from=builder /install /usr/local

# 复制应用代码
COPY src/ src/

# 设置环境变量
ENV PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1 \
    PATH="/app/.local/bin:${PATH}"

USER appuser

EXPOSE 8000

CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"]

# .github/workflows/docker.yml
name: Docker Build & Push

on:
  push:
    branches: [main]
    tags: ["v*"]

jobs:
  docker:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    steps:
      - uses: actions/checkout@v4

      - name: 设置 QEMU（多平台模拟）
        uses: docker/setup-qemu-action@v3

      - name: 设置 Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: 登录 GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: 登录 Docker Hub（可选）
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: 提取元数据
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ghcr.io/${{ github.repository }}
            docker.io/myuser/myapp
          tags: |
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha,prefix=

      - name: 构建并推送
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            VERSION=${{ github.ref_name }}

.git
.github
__pycache__
*.pyc
.pytest_cache
.mypy_cache
.ruff_cache
.venv
venv
*.egg-info
dist
build
docs
tests
*.md
!README.md

mylib/
├── pyproject.toml
├── README.md
├── CHANGELOG.md
├── changelog.d/
│   ├── 42.feature.md
│   └── 57.bugfix.md
├── src/
│   └── mylib/
│       ├── __init__.py
│       └── core.py
├── tests/
│   ├── conftest.py
│   └── test_core.py
└── .github/
    └── workflows/
        ├── ci.yml
        └── release.yml

[project]
name = "mylib"
dynamic = ["version"]
description = "A demonstration library"
readme = "README.md"
requires-python = ">=3.10"
license = "MIT"
authors = [{ name = "Your Name", email = "you@example.com" }]
classifiers = [
    "Development Status :: 4 - Beta",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
]

[project.optional-dependencies]
dev = ["ruff", "mypy"]
test = ["pytest", "pytest-cov"]

[tool.setuptools_scm]
fallback_version = "0.0.0"

[tool.towncrier]
package = "mylib"
directory = "changelog.d"
filename = "CHANGELOG.md"

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

jobs:
  quality:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install --quiet ruff mypy -e ".[dev]"
      - run: ruff check .
      - run: ruff format --check .
      - run: mypy src/ --ignore-missing-imports

  test:
    needs: quality
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - run: pip install --quiet -e ".[test]"
      - run: pytest --cov=mylib --cov-report=xml -v
      - if: matrix.python-version == '3.12'
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

name: Release

on:
  push:
    tags: ["v*"]

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install --quiet build twine
      - run: python -m build
      - run: twine check dist/*
      - uses: actions/upload-artifact@v4
        with:
          name: dist
          path: dist/

  publish:
    needs: build
    runs-on: ubuntu-latest
    environment: pypi
    permissions:
      id-token: write
      contents: write
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: dist/

      - name: 发布到 PyPI
        uses: pypa/gh-action-pypi-publish@release/v1

      - name: 创建 GitHub Release
        uses: softprops/action-gh-release@v2
        with:
          generate_release_notes: true
          files: dist/*

myapi/
├── pyproject.toml
├── Dockerfile
├── docker-compose.yml
├── docker-compose.prod.yml
├── .env.example
├── src/
│   └── myapi/
│       ├── __init__.py
│       ├── main.py
│       ├── config.py
│       ├── routers/
│       │   └── users.py
│       └── models.py
├── tests/
│   ├── conftest.py
│   └── test_users.py
├── alembic/
│   └── versions/
├── alembic.ini
└── .github/
    └── workflows/
        ├── ci.yml
        └── deploy.yml

# ── 构建阶段 ──
FROM python:3.12-slim AS builder

WORKDIR /build

# 安装系统依赖（如 PostgreSQL 客户端库）
RUN apt-get update && \
    apt-get install -y --no-install-recommends libpq-dev gcc && \
    rm -rf /var/lib/apt/lists/*

COPY pyproject.toml README.md ./
COPY src/ src/

RUN pip install --no-cache-dir --prefix=/install ".[prod]"

# ── 运行阶段 ──
FROM python:3.12-slim AS runtime

RUN apt-get update && \
    apt-get install -y --no-install-recommends libpq5 && \
    rm -rf /var/lib/apt/lists/* && \
    groupadd -r appuser && useradd -r -g appuser appuser

WORKDIR /app

COPY --from=builder /install /usr/local
COPY src/ src/
COPY alembic/ alembic/
COPY alembic.ini .

ENV PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1

USER appuser

EXPOSE 8000

HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
    CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"

CMD ["uvicorn", "src.myapi.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4"]

version: "3.9"

services:
  api:
    image: ghcr.io/myuser/myapi:latest
    container_name: myapi
    restart: unless-stopped
    ports:
      - "8000:8000"
    env_file:
      - .env
    depends_on:
      db:
        condition: service_healthy
      redis:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"]
      interval: 30s
      timeout: 5s
      retries: 3

  db:
    image: postgres:16-alpine
    restart: unless-stopped
    volumes:
      - pgdata:/var/lib/postgresql/data
    environment:
      POSTGRES_USER: ${DB_USER}
      POSTGRES_PASSWORD: ${DB_PASSWORD}
      POSTGRES_DB: ${DB_NAME}
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${DB_USER}"]
      interval: 10s
      timeout: 5s
      retries: 5

  redis:
    image: redis:7-alpine
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 5s
      retries: 5

volumes:
  pgdata:

name: Deploy

on:
  push:
    branches: [main]

jobs:
  test:
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_USER: test
          POSTGRES_PASSWORD: test
          POSTGRES_DB: testdb
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U test"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install --quiet -e ".[test]"
      - run: pytest -v
        env:
          DATABASE_URL: postgresql://test:test@localhost:5432/testdb

  docker-build-push:
    needs: test
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    steps:
      - uses: actions/checkout@v4

      - uses: docker/setup-buildx-action@v3

      - uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: |
            ghcr.io/${{ github.repository }}:${{ github.sha }}
            ghcr.io/${{ github.repository }}:latest
          cache-from: type=gha
          cache-to: type=gha,mode=max

  deploy:
    needs: docker-build-push
    runs-on: ubuntu-latest
    environment: production
    steps:
      - name: 部署到生产服务器
        uses: appleboy/ssh-action@v1
        with:
          host: ${{ secrets.SERVER_HOST }}
          username: ${{ secrets.SERVER_USER }}
          key: ${{ secrets.SERVER_SSH_KEY }}
          script: |
            cd /opt/myapi
            docker compose -f docker-compose.prod.yml pull
            docker compose -f docker-compose.prod.yml up -d --remove-orphans
            docker image prune -f

      - name: 健康检查
        run: |
          sleep 10
          curl -f https://app.example.com/health || exit 1

      - name: 通知部署结果
        if: always()
        run: |
          STATUS="${{ job.status }}"
          curl -X POST "${{ secrets.NOTIFICATION_WEBHOOK }}" \
            -H "Content-Type: application/json" \
            -d "{\"text\": \"部署 ${{ github.repository }} 到生产环境: $STATUS\"}"