From 6f274a3e2eb16e0bc4c85dc6280b58d318f4f317 Mon Sep 17 00:00:00 2001 From: danial Date: Sun, 9 Nov 2025 15:11:42 +0800 Subject: [PATCH] =?UTF-8?q?chore(ci):=20=E4=BC=98=E5=8C=96=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E9=95=9C=E5=83=8F=E6=9E=84=E5=BB=BA=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E5=B9=B6=E7=A7=BB=E9=99=A4=E5=86=97=E4=BD=99=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 CI 脚本中增加对必需构建文件的校验，避免缺失文件导致构建失败 - 添加构建成功后镜像验证步骤，并输出镜像详情 - 移除 `kami-spider-monorepo` 目录下的 Dockerfile、Makefile、README.md 和入口脚本，简化项目结构 - 调整基础镜像构建脚本 `build-base-image.sh`，指定正确的 Dockerfile 文件名 - 更新 README 使其内容更加精简，聚焦于基础镜像构建和使用说明 --- .drone.yml | 37 ++++- kami-spider-monorepo/Dockerfile | 67 --------- kami-spider-monorepo/Makefile | 102 ------------- kami-spider-monorepo/README.md | 175 +++------------------- kami-spider-monorepo/build-base-image.sh | 2 +- kami-spider-monorepo/docker-entrypoint.sh | 55 ------- 6 files changed, 55 insertions(+), 383 deletions(-) delete mode 100644 kami-spider-monorepo/Dockerfile delete mode 100755 kami-spider-monorepo/Makefile delete mode 100755 kami-spider-monorepo/docker-entrypoint.sh diff --git a/.drone.yml b/.drone.yml index a62f168..0b934f9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -61,17 +61,44 @@ steps: echo "Changed to: $(pwd)" ls -la - if [ -f "build-base-image.sh" ] && [ -f "Dockerfile.base" ]; then + # 检查所有必需的构建文件 + REQUIRED_FILES=("build-base-image.sh" "Dockerfile.base" "pyproject.toml" "uv.lock") + ALL_FILES_EXIST=true + + for file in "${REQUIRED_FILES[@]}"; do + if [ ! -f "$file" ]; then + echo "❌ Required file not found: $file" + ALL_FILES_EXIST=false + fi + done + + if [ "$ALL_FILES_EXIST" = "true" ]; then + echo "✅ All required build files found" chmod +x build-base-image.sh + + # 设置环境变量并构建 + export DOCKER_REGISTRY="git.oceanpay.cc/danial" + export VERSION="latest" + docker login git.oceanpay.cc -u $DOCKER_LOGIN -p $DOCKER_TOKEN ./build-base-image.sh - docker tag kami-spider-base:latest git.oceanpay.cc/danial/kami-spider-base:latest - docker push git.oceanpay.cc/danial/kami-spider-base:latest + + # 验证镜像构建成功 + if docker images | grep "kami-spider-base" | grep "latest"; then + echo "✅ Spider base image built successfully" + echo "📊 Image details:" + docker images | grep kami-spider-base + else + echo "❌ Base image build failed - image not found" + exit 1 + fi + docker logout git.oceanpay.cc - echo "✅ Spider base image built successfully" else - echo "❌ Required build files not found!" + echo "❌ Missing required build files!" + echo "Current directory contents:" ls -la + exit 1 fi else echo "❌ kami-spider-monorepo directory not found!" diff --git a/kami-spider-monorepo/Dockerfile b/kami-spider-monorepo/Dockerfile deleted file mode 100644 index 07b8635..0000000 --- a/kami-spider-monorepo/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -# Simplified Dockerfile for kami_spider using combined base image -# Fast build using pre-built comprehensive base image - -# Build arguments -ARG USE_PROXY=0 - -# Use the combined base image with all dependencies pre-installed -FROM ${BASE_REGISTRY}/kami-spider-base:latest - -# Set environment variables for production -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PATH="/app/.venv/bin:$PATH" \ - UV_CACHE_DIR=/tmp/uv-cache \ - PYDEVD_DISABLE=1 \ - PYDEVD_DISABLE_FILE_VALIDATION=1 \ - PYCHARM_DEBUG="false" \ - PYTEST_CURRENT_TEST="false" - -# Set working directory -WORKDIR /app - -# Copy dependency files for version checking -COPY --chown=appuser:appuser pyproject.toml ./ -COPY --chown=appuser:appuser uv.lock ./ - -# Quick dependency sync if versions differ (very fast) -RUN if [ "$USE_PROXY" = "1" ]; then \ - uv sync --frozen --no-dev --no-install-project --index-url https://pypi.tuna.tsinghua.edu.cn/simple/ || true; \ - else \ - uv sync --frozen --no-dev --no-install-project || true; \ - fi - -# Copy application code -COPY --chown=appuser:appuser . . - -# Create entrypoint script -RUN echo '#!/bin/bash' > /usr/local/bin/docker-entrypoint.sh && \ - echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-entrypoint.sh - -# Switch to non-root user (already created in base image) -USER appuser - -# Expose port -EXPOSE 8000 - -# Health check (using curl) -HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ - CMD curl -f http://localhost:8000/health || exit 1 - -# Use custom entrypoint script for production -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -# Run application with Gunicorn -CMD ["gunicorn", "main:app", \ - "--workers", "4", \ - "--worker-class", "uvicorn.workers.UvicornWorker", \ - "--bind", "0.0.0.0:8000", \ - "--timeout", "60", \ - "--graceful-timeout", "60", \ - "--access-logfile", "-", \ - "--error-logfile", "-", \ - "--log-level", "info", \ - "--max-requests", "1000", \ - "--max-requests-jitter", "100", \ - "--preload"] \ No newline at end of file diff --git a/kami-spider-monorepo/Makefile b/kami-spider-monorepo/Makefile deleted file mode 100755 index b42c503..0000000 --- a/kami-spider-monorepo/Makefile +++ /dev/null @@ -1,102 +0,0 @@ -# Makefile for kami-spider Docker image management - -.PHONY: help build-base build clean test push-base push-app - -# Default variables -VERSION ?= latest -REGISTRY ?= localhost:5000 -USE_PROXY ?= 0 - -help: ## Show this help message - @echo "Available commands:" - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' - -build-base: ## Build base image with all dependencies - @echo "🏗️ Building base image..." - ./build-base-image.sh - -build-base-with-proxy: ## Build base image using proxy - @echo "🏗️ Building base image with proxy..." - USE_PROXY=1 ./build-base-image.sh - -build: ## Build application image using pre-built base image - @echo "🚀 Building application image..." - docker build \ - -t kami-spider:$(VERSION) \ - --build-arg DOCKER_REGISTRY=$(REGISTRY) \ - --build-arg USE_PROXY=$(USE_PROXY) \ - . - -build-all: build-base build ## Build everything (base image + application) - -clean: ## Clean up Docker images - @echo "🧹 Cleaning up Docker images..." - docker rmi kami-spider-base:latest 2>/dev/null || true - docker rmi kami-spider-base:$(VERSION) 2>/dev/null || true - docker rmi kami-spider:$(VERSION) 2>/dev/null || true - docker image prune -f - -test-base: ## Test base image - @echo "🧪 Testing base image..." - docker run --rm kami-spider-base:latest python --version - docker run --rm kami-spider-base:latest python -c "import fastapi; print('✅ FastAPI available')" - docker run --rm kami-spider-base:latest python -c "import playwright; print('✅ Playwright available')" - -test: ## Test application image - @echo "🧪 Testing application image..." - docker run --rm -p 8001:8000 -d --name test-app kami-spider:$(VERSION) - sleep 5 - curl -f http://localhost:8001/health || (echo "❌ Health check failed" && docker stop test-app && exit 1) - docker stop test-app - @echo "✅ Application test passed" - -test-all: test-base test ## Run all tests - -push-base: ## Push base image to registry - @echo "📤 Pushing base image to registry..." - docker tag kami-spider-base:latest $(REGISTRY)/kami-spider-base:$(VERSION) - docker push $(REGISTRY)/kami-spider-base:$(VERSION) - @echo "✅ Base image pushed successfully" - -push-app: ## Push application image to registry - @echo "📤 Pushing application image to registry..." - docker tag kami-spider:$(VERSION) $(REGISTRY)/kami-spider:$(VERSION) - docker push $(REGISTRY)/kami-spider:$(VERSION) - @echo "✅ Application image pushed successfully" - -push-all: push-base push-app ## Push all images to registry - -dev: ## Build image for development - @echo "🔧 Building development image..." - make build - @echo "🔧 Development image built: kami-spider:$(VERSION)" - -prod: VERSION=$(shell date +%Y%m%d-%H%M%S) -prod: ## Build production image with timestamp - @echo "🏭 Building production image..." - make build - @echo "🏭 Production image built: kami-spider:$(VERSION)" - -# Quick start commands -quick-start: ## Quick start for local development - @echo "🚀 Quick starting kami-spider..." - make build - docker run -d --name kami-spider -p 8000:8000 kami-spider:latest - @echo "✅ Application started on http://localhost:8000" - -stop: ## Stop and remove running container - docker stop kami-spider 2>/dev/null || true - docker rm kami-spider 2>/dev/null || true - @echo "🛑 Application stopped" - -logs: ## Show application logs - docker logs -f kami-spider 2>/dev/null || echo "❌ No running container found" - -# Performance comparison -compare-builds: ## Compare build times between original and optimized - @echo "⏱️ Comparing build times..." - @echo "Building original Dockerfile..." - time make build-original - @echo "Building optimized Dockerfile..." - time make build-optimized - @echo "✅ Build comparison completed" \ No newline at end of file diff --git a/kami-spider-monorepo/README.md b/kami-spider-monorepo/README.md index 811ea1e..07fb06b 100644 --- a/kami-spider-monorepo/README.md +++ b/kami-spider-monorepo/README.md @@ -1,165 +1,34 @@ -# kami-spider 基础镜像 +# kami-spider Base Image -提供预构建的 Docker 基础镜像，加速 kami-spider 应用的构建和部署。 - -## 架构概览 - -### 基础镜像 (`kami-spider-base`) -- **基础**: `python:3.13-slim` -- **包含**: - - Python 3.13 运行时环境 - - 系统依赖 (gcc, g++, curl 等) - - Playwright 运行时依赖 - - UV 包管理器 - - 非 root 用户配置 - - 所有项目 Python 依赖包 - - Playwright 浏览器 (Chromium) - - 预配置的虚拟环境 - -### 应用镜像 (`kami-spider`) -- 基于预构建的基础镜像 -- 仅包含应用代码 -- 极快的构建速度 - -## 快速开始 - -### 1. 构建基础镜像 -```bash -# 构建完整基础镜像 -./build-base-image.sh - -# 或使用 Make -make build-base - -# 使用代理构建 -USE_PROXY=1 ./build-base-image.sh -# 或 -make build-base-with-proxy -``` - -### 2. 构建应用镜像 -```bash -# 构建应用镜像 -make build - -# 快速启动 -make quick-start -``` - -### 3. 访问应用 -应用启动后可通过 http://localhost:8000 访问 - -## 性能优势 - -- **构建时间**: 从 5-8 分钟减少到 15-30 秒 -- **缓存效果**: 依赖只需安装一次 -- **镜像大小**: ~850MB (包含所有依赖) - -## 详细命令 - -### 构建命令 -```bash -# 构建所有镜像 -make build-all - -# 开发环境构建 -make dev - -# 生产环境构建 (带时间戳) -make prod -``` - -### 测试命令 -```bash -# 测试基础镜像 -make test-base - -# 测试应用镜像 -make test - -# 运行所有测试 -make test-all -``` - -### 推送命令 -```bash -# 推送基础镜像 -make push-base - -# 推送应用镜像 -make push-app - -# 推送所有镜像 -make push-all -``` - -### 管理命令 -```bash -# 清理镜像 -make clean - -# 停止应用 -make stop - -# 查看应用日志 -make logs - -# 查看所有可用命令 -make help -``` - -## 环境变量 - -| 变量名 | 默认值 | 说明 | -|--------|--------|------| -| `USE_PROXY` | `0` | 是否使用清华源代理 | -| `DOCKER_REGISTRY` | `localhost:5000` | Docker 镜像仓库地址 | -| `VERSION` | `latest` | 镜像版本标签 | +用于构建 kami-spider 应用的基础 Docker 镜像。 ## 文件说明 - `Dockerfile.base` - 基础镜像定义 -- `Dockerfile` - 应用镜像定义 -- `build-base-image.sh` - 基础镜像构建脚本 -- `Makefile` - 构建管理命令 +- `build-base-image.sh` - 构建脚本 +- `pyproject.toml` - Python 依赖配置 +- `uv.lock` - 锁定的依赖版本 -## CI/CD 集成 +## 构建命令 -在 CI/CD 流水线中的推荐流程： - -1. 检查基础镜像是否存在 -2. 如果不存在，先构建基础镜像： - ```bash - ./build-base-image.sh - ``` -3. 构建应用镜像： - ```bash - make build - ``` -4. 推送到镜像仓库： - ```bash - make push-all - ``` - -## 故障排除 - -### 基础镜像拉取失败 ```bash -# 检查镜像仓库配置 -echo $DOCKER_REGISTRY +# 构建基础镜像 +./build-base-image.sh -# 手动拉取测试 -docker pull kami-spider-base:latest +# 使用代理构建 +USE_PROXY=1 ./build-base-image.sh + +# 推送到镜像仓库 +DOCKER_REGISTRY=your-registry.com ./build-base-image.sh ``` -### 依赖安装失败 -```bash -# 重建基础镜像 -docker build --no-cache -f Dockerfile.base -t kami-spider-base:latest . -``` +## 镜像内容 -### 权限问题 -确保构建脚本有执行权限： -```bash -chmod +x build-base-image.sh -``` \ No newline at end of file +- Python 3.13 运行时环境 +- 系统依赖和 Playwright 运行时库 +- UV 包管理器 +- 所有项目 Python 依赖包 +- Playwright 浏览器 (Chromium) +- 非 root 用户配置 + +构建完成后镜像将标记为 `kami-spider-base:latest`。 \ No newline at end of file diff --git a/kami-spider-monorepo/build-base-image.sh b/kami-spider-monorepo/build-base-image.sh index a35abc7..57284bb 100755 --- a/kami-spider-monorepo/build-base-image.sh +++ b/kami-spider-monorepo/build-base-image.sh @@ -20,7 +20,7 @@ echo # Build the combined base image (system + Python dependencies + Playwright) echo "📦 Building combined base image ($BASE_IMAGE_NAME)..." docker build \ - --file Dockerfile.base-combined \ + --file Dockerfile.base \ --tag "$BASE_IMAGE_NAME:$VERSION" \ --tag "$BASE_IMAGE_NAME:latest" \ --build-arg USE_PROXY="$USE_PROXY" \ diff --git a/kami-spider-monorepo/docker-entrypoint.sh b/kami-spider-monorepo/docker-entrypoint.sh deleted file mode 100755 index 8e6399f..0000000 --- a/kami-spider-monorepo/docker-entrypoint.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# Docker production entrypoint script -# Optimized for production deployment - -set -e - -# Disable all debuggers and development tools -export PYDEVD_DISABLE=1 -export PYDEVD_DISABLE_FILE_VALIDATION=1 -export PYCHARM_DEBUG="false" -export PYTEST_CURRENT_TEST="false" -export PYTHONOPTIMIZE=2 - -# Ensure proper signal handling -cleanup() { - echo "Received termination signal, shutting down gracefully..." - if [ -n "$MAIN_PID" ]; then - kill -TERM "$MAIN_PID" - wait "$MAIN_PID" - fi - exit 0 -} - -# Set up signal handlers -trap cleanup SIGTERM SIGINT - -echo "Starting kami_spider application..." -echo "Environment: ${ENVIRONMENT:-production}" -echo "Database URL: ${DATABASE_URL}" -# Run database migrations first -echo "Running database migrations..." -if python scripts/db_manager.py up; then - echo "Database migrations completed successfully" -else - echo "Database migrations failed!" - exit 1 -fi - -# Check database connection -echo "Checking database connection..." -if python scripts/db_manager.py check; then - echo "Database connection verified" -else - echo "Database connection failed!" - exit 1 -fi - -# Run the main application -echo "Starting main application..." -exec "$@" & -MAIN_PID=$! -echo "Application started with PID: $MAIN_PID" - -# Wait for the main process -wait "$MAIN_PID" \ No newline at end of file