From 48cdcb61405cf52f9de4ea11910f03c5210e00e2 Mon Sep 17 00:00:00 2001 From: danial Date: Thu, 11 Sep 2025 17:57:18 +0800 Subject: [PATCH] feat: Add deployment scripts and configuration for Apple Gift Card Exchange Platform - Create README.md for deployment instructions including environment requirements and setup steps. - Implement deploy.sh script for automated deployment of development and production environments. - Add combined Docker Compose configuration for frontend and backend services. - Include Redis configuration file for optimized memory management and persistence. - Update frontend Dockerfile to handle Next.js asset paths and static files. - Remove obsolete deployment files and configurations from frontend directory. --- .claude/settings.local.json | 21 ++ .gitignore | 13 +- CLAUDE.md | 172 +++++++++ CLAUDE_CN.md | 172 +++++++++ README.md | 328 ++++++++++++++++ backend/.claude/settings.local.json | 9 +- backend/.env.example | 1 - backend/.env.local | 9 +- backend/.env.production | 13 +- backend/Dockerfile | 39 ++ backend/Dockerfile.test | 11 + backend/Dockerfile.worker | 41 ++ .../alembic/versions/002_add_batch_tasks.py | 173 --------- backend/alembic/versions/add_links_table.py | 42 --- backend/app/core/celery_app.py | 7 + backend/app/core/config.py | 8 +- backend/app/tasks/__init__.py | 5 +- backend/app/tasks/crawler_tasks.py | 4 +- backend/deploy/Dockerfile | 98 ----- backend/deploy/helm/apple-exchange/Chart.yaml | 9 - backend/deploy/helm/apple-exchange/README.md | 129 ------- .../helm/apple-exchange/templates/NOTES.txt | 27 -- .../apple-exchange/templates/_helpers.tpl | 62 ---- .../apple-exchange/templates/configmap.yaml | 22 -- .../templates/deployment-api.yaml | 127 ------- .../templates/deployment-beat.yaml | 83 ----- .../templates/deployment-flower.yaml | 80 ---- .../templates/deployment-worker.yaml | 130 ------- .../helm/apple-exchange/templates/hpa.yaml | 67 ---- .../apple-exchange/templates/ingress.yaml | 52 --- .../helm/apple-exchange/templates/pvc.yaml | 53 --- .../helm/apple-exchange/templates/secret.yaml | 11 - .../apple-exchange/templates/service.yaml | 41 -- .../templates/serviceaccount.yaml | 12 - .../deploy/helm/apple-exchange/values.yaml | 132 ------- backend/deploy/k8s/configmap.yaml | 87 ----- backend/deploy/k8s/deployment.yaml | 349 ------------------ backend/deploy/k8s/hpa.yaml | 83 ----- backend/deploy/k8s/namespace.yaml | 7 - backend/deploy/k8s/nodeport-access.md | 34 -- backend/deploy/k8s/persistent-volumes.yaml | 121 ------ backend/deploy/k8s/secret.yaml | 13 - backend/deploy/k8s/service.yaml | 117 ------ backend/deploy/redis.conf | 61 --- backend/docker-entrypoint-simple.sh | 5 + backend/{deploy => }/docker-entrypoint.sh | 16 +- backend/pyproject.toml | 162 ++++---- backend/run.py | 110 ++++++ backend/test_gunicorn.py | 88 +++++ backend/test_tasks.py | 56 +++ backend/uv.lock | 128 +++++-- deploy/README.md | 272 ++++++++++++++ deploy/deploy.sh | 204 ++++++++++ .../docker-compose.combined.yml | 207 +++++++---- deploy/redis.conf | 42 +++ frontend/.dockerignore | 64 +++- frontend/Dockerfile | 14 + frontend/deploy/.dockerignore | 37 -- frontend/deploy/Dockerfile | 57 --- frontend/deploy/k8s/configmap.yaml | 8 - frontend/deploy/k8s/deployment.yaml | 56 --- frontend/deploy/k8s/ingress.yaml | 25 -- frontend/deploy/k8s/service.yaml | 15 - frontend/deploy/nginx.conf | 128 ------- 64 files changed, 1986 insertions(+), 2783 deletions(-) create mode 100644 .claude/settings.local.json create mode 100644 CLAUDE.md create mode 100644 CLAUDE_CN.md create mode 100644 README.md create mode 100644 backend/Dockerfile create mode 100644 backend/Dockerfile.test create mode 100644 backend/Dockerfile.worker delete mode 100644 backend/alembic/versions/002_add_batch_tasks.py delete mode 100644 backend/alembic/versions/add_links_table.py delete mode 100644 backend/deploy/Dockerfile delete mode 100644 backend/deploy/helm/apple-exchange/Chart.yaml delete mode 100644 backend/deploy/helm/apple-exchange/README.md delete mode 100644 backend/deploy/helm/apple-exchange/templates/NOTES.txt delete mode 100644 backend/deploy/helm/apple-exchange/templates/_helpers.tpl delete mode 100644 backend/deploy/helm/apple-exchange/templates/configmap.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/deployment-api.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/deployment-beat.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/deployment-flower.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/deployment-worker.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/hpa.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/ingress.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/pvc.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/secret.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/service.yaml delete mode 100644 backend/deploy/helm/apple-exchange/templates/serviceaccount.yaml delete mode 100644 backend/deploy/helm/apple-exchange/values.yaml delete mode 100644 backend/deploy/k8s/configmap.yaml delete mode 100644 backend/deploy/k8s/deployment.yaml delete mode 100644 backend/deploy/k8s/hpa.yaml delete mode 100644 backend/deploy/k8s/namespace.yaml delete mode 100644 backend/deploy/k8s/nodeport-access.md delete mode 100644 backend/deploy/k8s/persistent-volumes.yaml delete mode 100644 backend/deploy/k8s/secret.yaml delete mode 100644 backend/deploy/k8s/service.yaml delete mode 100644 backend/deploy/redis.conf create mode 100644 backend/docker-entrypoint-simple.sh rename backend/{deploy => }/docker-entrypoint.sh (73%) create mode 100644 backend/run.py create mode 100644 backend/test_gunicorn.py create mode 100644 backend/test_tasks.py create mode 100644 deploy/README.md create mode 100644 deploy/deploy.sh rename backend/deploy/docker-compose.yml => deploy/docker-compose.combined.yml (69%) create mode 100644 deploy/redis.conf delete mode 100644 frontend/deploy/.dockerignore delete mode 100644 frontend/deploy/Dockerfile delete mode 100644 frontend/deploy/k8s/configmap.yaml delete mode 100644 frontend/deploy/k8s/deployment.yaml delete mode 100644 frontend/deploy/k8s/ingress.yaml delete mode 100644 frontend/deploy/k8s/service.yaml delete mode 100644 frontend/deploy/nginx.conf diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..289d2c4 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,21 @@ +{ + "permissions": { + "allow": [ + "Bash(docker-compose:*)", + "Bash(uv sync:*)", + "Bash(uv run:*)", + "Bash(pip install:*)", + "Bash(uv add:*)", + "Bash(docker build:*)", + "Bash(docker port:*)", + "Bash(docker inspect:*)", + "Bash(docker exec:*)", + "Bash(docker run:*)", + "Bash(curl:*)", + "Bash(docker logs:*)", + "Bash(docker rm:*)" + ], + "deny": [], + "ask": [] + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index eefa2f4..4fca9cb 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,15 @@ /backend/__pycache__/ /backend/templates/ /.idea/ -/.vscode/ \ No newline at end of file +/.vscode/ +/.claude/ +/.codebuddy/ +/.idea/ +/backend/.env/ +/frontend/node_modules/ +/frontend/dist/ +/frontend/.next/ +/frontend/out/ +/backend/.codebuddy/ +/frontend/.codebuddy/ +/frontned/.claude/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5cdece1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,172 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Development Commands + +### Frontend (Next.js) +- `cd frontend && bun install` - Install dependencies +- `cd frontend && bun run dev` - Start development server with Turbopack +- `cd frontend && bun run build` - Build production version +- `cd frontend && bun run start` - Start production server +- `cd frontend && bun run lint` - Run ESLint checks +- `cd frontend && bun run generate:api` - Generate API client from OpenAPI spec +- `cd frontend && bun run generate:api:watch` - Generate API client with watch mode + +### Backend (FastAPI) +- `cd backend && uv sync` - Install dependencies with UV +- `cd backend && uv run python app/main.py` - Run development server +- `cd backend && uv run pytest` - Run tests +- `cd backend && uv run black .` - Format code with Black +- `cd backend && uv run isort .` - Sort imports with isort +- `cd backend && uv run mypy .` - Type checking with MyPy +- `cd backend && uv run ruff check .` - Lint with Ruff + +### Docker Development +- `cd backend && docker-compose up -d` - Start all services with Docker Compose +- `cd backend && docker-compose down` - Stop all services +- `cd backend && docker-compose logs -f` - View logs + +## Architecture Overview + +### Project Structure +This is a full-stack application with: +- **Frontend**: Next.js 15 with App Router, React 18, TypeScript +- **Backend**: FastAPI with Python 3.13, async/await architecture +- **Infrastructure**: Docker, Kubernetes, Redis, PostgreSQL + +### Frontend Architecture +- **Framework**: Next.js 15 with App Router +- **UI**: Apple-style design system with custom components +- **State Management**: React Context + TanStack Query for server state +- **Forms**: React Hook Form + Zod validation +- **HTTP**: Axios with enhanced client +- **Styling**: Tailwind CSS + Less for complex styles +- **Animation**: Framer Motion +- **Package Manager**: Bun (configured in bunfig.toml) + +### Backend Architecture +- **Framework**: FastAPI with async/await +- **Architecture**: Distributed microservices with Celery task queues +- **Browser Automation**: Playwright for web scraping +- **Database**: PostgreSQL with asyncpg +- **Cache**: Redis for distributed locking and state management +- **Task Queue**: Celery with Redis broker +- **Monitoring**: OpenTelemetry integration +- **Package Manager**: UV (configured in pyproject.toml) + +### Key Directories + +#### Frontend (`frontend/`) +- `src/app/` - Next.js App Router pages +- `src/components/` - Reusable components + - `dashboard/` - Main dashboard components + - `forms/` - Form components with validation + - `layout/` - Layout and theme components + - `ui/` - Base UI components (Apple-style) + - `animate-ui/` - Animated UI components +- `src/lib/` - Utility libraries + - `api/` - API client and generated types + - `contexts/` - React contexts (theme, refresh) + - `hooks/` - Custom React hooks + - `telemetry/` - OpenTelemetry integration + +#### Backend (`backend/`) +- `app/api/` - FastAPI routes and endpoints +- `app/core/` - Core functionality (config, database, redis, etc.) +- `app/models/` - SQLAlchemy data models +- `app/schemas/` - Pydantic schemas +- `app/services/` - Business logic services +- `app/tasks/` - Celery task definitions +- `app/repositories/` - Data access layer +- `deploy/` - Docker and deployment configurations +- `docs/` - Architecture and project documentation + +### Core Features + +#### Frontend Features +- Real-time dashboard with auto-refresh +- Apple-style UI with smooth animations +- File upload and data management +- Order and task monitoring +- Link management system +- Theme switching (dark/light) +- OpenTelemetry telemetry integration + +#### Backend Features +- Distributed web scraping with Playwright +- Redis-based distributed locking +- Celery task queue system +- Automatic task recovery and failure handling +- Kubernetes deployment support +- Graceful shutdown handling +- Comprehensive health checks +- Structured logging with OpenTelemetry + +### API Integration +- Backend provides OpenAPI specification at `http://localhost:8000/openapi.json` +- Frontend auto-generates API client code using Orval +- Generated files in `frontend/src/lib/api/generated/` +- Enhanced axios client with interceptors and error handling + +### State Management + +#### Frontend State +- **Global State**: React Context for theme and refresh settings +- **Server State**: TanStack Query for API data management +- **Local State**: React hooks for component state +- **Form State**: React Hook Form with Zod schemas + +#### Backend State +- **Database**: PostgreSQL with SQLAlchemy ORM +- **Cache**: Redis for distributed state and locking +- **Task State**: Celery backend for task progress tracking +- **Session State**: Redis-based session management + +### Deployment Configuration + +#### Development +- Frontend: Bun development server +- Backend: UV development server +- Services: Docker Compose for Redis, PostgreSQL + +#### Production +- Frontend: Docker container with Nginx +- Backend: Multi-service Docker deployment +- Orchestration: Kubernetes with Helm charts +- Monitoring: Prometheus + Grafana +- Logging: Structured logs with OpenTelemetry + +### Development Notes + +#### Environment Variables +**Frontend** (`.env.local`): +- `NEXT_PUBLIC_API_URL` - Backend API URL +- `NEXT_PUBLIC_ENV` - Environment (development/production) +- `NEXT_PUBLIC_TELEMETRY_ENDPOINT` - OpenTelemetry endpoint + +**Backend** (`.env`): +- `REDIS_URL` - Redis connection URL +- `DATABASE_URL` - PostgreSQL connection URL +- `SERVICE_TYPE` - Service type (api/worker/recovery) +- `LOG_LEVEL` - Logging level + +#### Code Quality +- **Frontend**: ESLint + Prettier + TypeScript +- **Backend**: Black + isort + MyPy + Ruff + pytest +- **Pre-commit**: Hooks for both frontend and backend +- **Testing**: pytest for backend, Jest setup available for frontend + +#### Key Architectural Patterns +- **Microservices**: Backend uses service-oriented architecture +- **Distributed Systems**: Redis-based coordination and locking +- **Event-Driven**: Celery task queues for async processing +- **Reactive UI**: Frontend uses real-time updates with polling +- **Type Safety**: Full TypeScript coverage on both ends + +#### Special Considerations +- **Distributed Locking**: Critical for multi-replica deployments +- **Graceful Shutdown**: Ensures task completion during deployments +- **Error Handling**: Comprehensive error recovery mechanisms +- **Monitoring**: Full observability with OpenTelemetry +- **Scaling**: Horizontal scaling support with Kubernetes HPA \ No newline at end of file diff --git a/CLAUDE_CN.md b/CLAUDE_CN.md new file mode 100644 index 0000000..f16dda1 --- /dev/null +++ b/CLAUDE_CN.md @@ -0,0 +1,172 @@ +# CLAUDE_CN.md + +本文件为 Claude Code (claude.ai/code) 在此代码库中工作时提供中文指导。 + +## 开发命令 + +### 前端 (Next.js) +- `cd frontend && bun install` - 安装依赖 +- `cd frontend && bun run dev` - 启动开发服务器（使用 Turbopack） +- `cd frontend && bun run build` - 构建生产版本 +- `cd frontend && bun run start` - 启动生产服务器 +- `cd frontend && bun run lint` - 运行 ESLint 检查 +- `cd frontend && bun run generate:api` - 从 OpenAPI 规范生成 API 客户端 +- `cd frontend && bun run generate:api:watch` - 监听模式生成 API 客户端 + +### 后端 (FastAPI) +- `cd backend && uv sync` - 使用 UV 安装依赖 +- `cd backend && uv run python app/main.py` - 运行开发服务器 +- `cd backend && uv run pytest` - 运行测试 +- `cd backend && uv run black .` - 使用 Black 格式化代码 +- `cd backend && uv run isort .` - 使用 isort 排序导入 +- `cd backend && uv run mypy .` - 使用 MyPy 进行类型检查 +- `cd backend && uv run ruff check .` - 使用 Ruff 进行代码检查 + +### Docker 开发 +- `cd backend && docker-compose up -d` - 使用 Docker Compose 启动所有服务 +- `cd backend && docker-compose down` - 停止所有服务 +- `cd backend && docker-compose logs -f` - 查看日志 + +## 架构概述 + +### 项目结构 +这是一个全栈应用程序，包含： +- **前端**: Next.js 15 with App Router, React 18, TypeScript +- **后端**: FastAPI with Python 3.13, async/await 架构 +- **基础设施**: Docker, Kubernetes, Redis, PostgreSQL + +### 前端架构 +- **框架**: Next.js 15 with App Router +- **UI**: 苹果风格设计系统，自定义组件 +- **状态管理**: React Context + TanStack Query 管理服务端状态 +- **表单**: React Hook Form + Zod 验证 +- **HTTP**: 增强版 Axios 客户端 +- **样式**: Tailwind CSS + Less 处理复杂样式 +- **动画**: Framer Motion +- **包管理器**: Bun (配置在 bunfig.toml 中) + +### 后端架构 +- **框架**: FastAPI with async/await +- **架构**: 分布式微服务，使用 Celery 任务队列 +- **浏览器自动化**: Playwright 网页爬虫 +- **数据库**: PostgreSQL with asyncpg +- **缓存**: Redis 分布式锁和状态管理 +- **任务队列**: Celery with Redis broker +- **监控**: OpenTelemetry 集成 +- **包管理器**: UV (配置在 pyproject.toml 中) + +### 核心目录 + +#### 前端 (`frontend/`) +- `src/app/` - Next.js App Router 页面 +- `src/components/` - 可重用组件 + - `dashboard/` - 主要仪表板组件 + - `forms/` - 表单组件（含验证） + - `layout/` - 布局和主题组件 + - `ui/` - 基础 UI 组件（苹果风格） + - `animate-ui/` - 动画 UI 组件 +- `src/lib/` - 工具库 + - `api/` - API 客户端和生成类型 + - `contexts/` - React 上下文（主题、刷新） + - `hooks/` - 自定义 React Hooks + - `telemetry/` - OpenTelemetry 集成 + +#### 后端 (`backend/`) +- `app/api/` - FastAPI 路由和端点 +- `app/core/` - 核心功能（配置、数据库、redis 等） +- `app/models/` - SQLAlchemy 数据模型 +- `app/schemas/` - Pydantic 模式 +- `app/services/` - 业务逻辑服务 +- `app/tasks/` - Celery 任务定义 +- `app/repositories/` - 数据访问层 +- `deploy/` - Docker 和部署配置 +- `docs/` - 架构和项目文档 + +### 核心功能 + +#### 前端功能 +- 实时仪表板，自动刷新 +- 苹果风格 UI，流畅动画 +- 文件上传和数据管理 +- 订单和任务监控 +- 链接管理系统 +- 主题切换（深色/浅色） +- OpenTelemetry 遥测集成 + +#### 后端功能 +- 使用 Playwright 分布式网页爬虫 +- 基于 Redis 的分布式锁 +- Celery 任务队列系统 +- 自动任务恢复和故障处理 +- Kubernetes 部署支持 +- 优雅关闭处理 +- 全面的健康检查 +- OpenTelemetry 结构化日志 + +### API 集成 +- 后端在 `http://localhost:8000/openapi.json` 提供 OpenAPI 规范 +- 前端使用 Orval 自动生成 API 客户端代码 +- 生成文件位于 `frontend/src/lib/api/generated/` +- 增强版 axios 客户端，包含拦截器和错误处理 + +### 状态管理 + +#### 前端状态 +- **全局状态**: React Context 管理主题和刷新设置 +- **服务端状态**: TanStack Query 管理 API 数据 +- **本地状态**: React Hooks 管理组件状态 +- **表单状态**: React Hook Form + Zod 模式 + +#### 后端状态 +- **数据库**: PostgreSQL with SQLAlchemy ORM +- **缓存**: Redis 分布式状态和锁 +- **任务状态**: Celery backend 任务进度跟踪 +- **会话状态**: Redis 基础会话管理 + +### 部署配置 + +#### 开发环境 +- 前端: Bun 开发服务器 +- 后端: UV 开发服务器 +- 服务: Docker Compose 管理 Redis, PostgreSQL + +#### 生产环境 +- 前端: Docker 容器 + Nginx +- 后端: 多服务 Docker 部署 +- 编排: Kubernetes with Helm charts +- 监控: Prometheus + Grafana +- 日志: OpenTelemetry 结构化日志 + +### 开发注意事项 + +#### 环境变量 +**前端** (`.env.local`): +- `NEXT_PUBLIC_API_URL` - 后端 API URL +- `NEXT_PUBLIC_ENV` - 环境（development/production） +- `NEXT_PUBLIC_TELEMETRY_ENDPOINT` - OpenTelemetry 端点 + +**后端** (`.env`): +- `REDIS_URL` - Redis 连接 URL +- `DATABASE_URL` - PostgreSQL 连接 URL +- `SERVICE_TYPE` - 服务类型（api/worker/recovery） +- `LOG_LEVEL` - 日志级别 + +#### 代码质量 +- **前端**: ESLint + Prettier + TypeScript +- **后端**: Black + isort + MyPy + Ruff + pytest +- **预提交**: 前后端都配置了 Git hooks +- **测试**: 后端使用 pytest，前端可配置 Jest + +#### 关键架构模式 +- **微服务**: 后端采用面向服务架构 +- **分布式系统**: Redis 协调和锁机制 +- **事件驱动**: Celery 任务队列异步处理 +- **响应式 UI**: 前端使用轮询实时更新 +- **类型安全**: 双端完整 TypeScript 覆盖 + +#### 特别注意事项 +- **分布式锁**: 多副本部署的关键机制 +- **优雅关闭**: 部署期间确保任务完成 +- **错误处理**: 全面的错误恢复机制 +- **监控**: OpenTelemetry 全链路可观测性 +- **扩展性**: Kubernetes HPA 水平扩展支持 \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..47d49de --- /dev/null +++ b/README.md @@ -0,0 +1,328 @@ +# Apple Gift Card Exchange Platform + +![License](https://img.shields.io/badge/license-MIT-blue.svg) +![Python](https://img.shields.io/badge/python-3.13+-blue.svg) +![Node.js](https://img.shields.io/badge/node-18+-green.svg) +![Docker](https://img.shields.io/badge/docker-ready-blue.svg) + +A modern, scalable platform for Apple gift card exchange with real-time monitoring, distributed processing, and Apple-style user interface. + +## 🌟 Features + +### Frontend (Next.js) +- **Apple-style Design**: Beautiful, modern UI following Apple Human Interface Guidelines +- **Real-time Dashboard**: Live data updates with configurable refresh intervals +- **Order Management**: Complete order lifecycle management with status tracking +- **Task Monitoring**: Real-time task status and control interface +- **Link Management**: Dynamic link management system for different amounts +- **Dark/Light Theme**: Full theme support with smooth transitions +- **Responsive Design**: Works seamlessly across all device sizes +- **Accessibility**: WCAG compliant with ARIA support + +### Backend (FastAPI) +- **Microservices Architecture**: Scalable distributed system design +- **Async Processing**: Full async/await support for high performance +- **Distributed Task Queue**: Celery with Redis for background processing +- **Browser Automation**: Playwright integration for Apple order processing +- **Real-time Monitoring**: OpenTelemetry observability stack +- **Database Management**: PostgreSQL with async SQLAlchemy ORM +- **Caching Layer**: Redis for distributed caching and state management +- **Health Checks**: Comprehensive health monitoring and graceful shutdown + +## 🏗️ Architecture + +### System Architecture +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Frontend │ │ Backend API │ │ Worker Pool │ +│ (Next.js) │◄──►│ (FastAPI) │◄──►│ (Celery) │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Browser │ │ PostgreSQL │ │ Playwright │ +│ │ │ Database │ │ Browsers │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ + ▼ + ┌─────────────────┐ + │ Redis │ + │ (Cache/Broker)│ + └─────────────────┘ +``` + +### Technology Stack + +#### Frontend +- **Framework**: Next.js 15 with App Router +- **Language**: TypeScript +- **Styling**: Tailwind CSS + SASS +- **UI Components**: Radix UI + Custom Apple-style components +- **State Management**: TanStack Query + React Context +- **Animations**: Framer Motion +- **Package Manager**: Bun + +#### Backend +- **Framework**: FastAPI +- **Language**: Python 3.13 +- **Database**: PostgreSQL with asyncpg +- **ORM**: SQLAlchemy 2.0 +- **Task Queue**: Celery with Redis +- **Browser Automation**: Playwright +- **Monitoring**: OpenTelemetry +- **Package Manager**: UV + +## 🚀 Quick Start + +### Prerequisites +- Python 3.13+ +- Node.js 18+ +- Docker & Docker Compose +- PostgreSQL +- Redis + +### Environment Setup + +1. **Clone the repository** +```bash +git clone https://github.com/your-org/apple-exchange.git +cd apple-exchange +``` + +2. **Backend Setup** +```bash +cd backend +uv sync +cp .env.example .env +# Configure your environment variables +``` + +3. **Frontend Setup** +```bash +cd frontend +bun install +cp .env.local.example .env.local +# Configure your environment variables +``` + +### Development Mode + +1. **Start Dependencies** +```bash +cd backend +docker-compose up -d redis postgres +``` + +2. **Start Backend** +```bash +cd backend +uv run python app/main.py +``` + +3. **Start Frontend** +```bash +cd frontend +bun run dev +``` + +4. **Start Workers** +```bash +cd backend +uv run celery -A app.core.celery_app worker --loglevel=info +uv run celery -A app.core.celery_app beat --loglevel=info +``` + +### Docker Development + +```bash +cd backend +docker-compose up -d +``` + +Access the application: +- Frontend: http://localhost:3000 +- Backend API: http://localhost:8000 +- API Documentation: http://localhost:8000/docs + +## 📚 API Documentation + +The backend provides comprehensive OpenAPI documentation: + +- **Swagger UI**: http://localhost:8000/docs +- **ReDoc**: http://localhost:8000/redoc +- **OpenAPI JSON**: http://localhost:8000/openapi.json + +### Key Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/api/v1/health` | GET | Health check | +| `/api/v1/orders` | GET/POST | Order management | +| `/api/v1/links` | GET/POST | Link management | +| `/api/v1/user-data` | GET/POST | User data management | +| `/api/v1/tasks` | GET | Task monitoring | + +## 🔧 Configuration + +### Backend Environment Variables +```env +# Database +DATABASE_URL=postgresql://user:password@localhost:5432/apple_exchange + +# Redis +REDIS_URL=redis://localhost:6379/0 + +# Application +SERVICE_TYPE=api +LOG_LEVEL=INFO +ENVIRONMENT=development + +# Security +SECRET_KEY=your-secret-key-here +``` + +### Frontend Environment Variables +```env +# API Configuration +NEXT_PUBLIC_API_URL=http://localhost:8000 +NEXT_PUBLIC_ENV=development + +# Monitoring +NEXT_PUBLIC_TELEMETRY_ENDPOINT=http://localhost:4318 +``` + +## 🏗️ Deployment + +### Production Deployment + +1. **Build Frontend** +```bash +cd frontend +bun run build +``` + +2. **Build Backend** +```bash +cd backend +docker build -t apple-exchange-backend . +``` + +3. **Kubernetes Deployment** +```bash +cd deploy/helm +helm install apple-exchange ./apple-exchange +``` + +### Docker Compose Production +```bash +cd backend +docker-compose -f docker-compose.yml up -d +``` + +## 📊 Monitoring + +### OpenTelemetry Integration +- **Distributed Tracing**: End-to-end request tracing +- **Metrics Collection**: Application and system metrics +- **Structured Logging**: JSON-formatted logs with context + +### Health Checks +- **Liveness Probes**: Container health monitoring +- **Readiness Probes**: Service availability checks +- **Database Connectivity**: Real-time database health monitoring + +### Monitoring Tools +- **Flower**: Celery task monitoring (http://localhost:5555) +- **Prometheus**: Metrics collection +- **Grafana**: Metrics visualization + +## 🧪 Testing + +### Backend Tests +```bash +cd backend +uv run pytest +uv run pytest --cov=app +uv run pytest -m "not slow" +``` + +### Frontend Tests +```bash +cd frontend +bun run test +bun run test:coverage +``` + +### Integration Tests +```bash +cd backend +uv run pytest -m integration +``` + +## 📝 Development Guidelines + +### Code Quality + +#### Backend +- **Formatting**: Black code formatter +- **Import Sorting**: isort +- **Type Checking**: MyPy +- **Linting**: Ruff +- **Testing**: pytest with coverage + +#### Frontend +- **Formatting**: Prettier +- **Linting**: ESLint +- **Type Checking**: TypeScript +- **Testing**: Jest + React Testing Library + +### Git Workflow +1. Create feature branch from `master` +2. Write code and tests +3. Run linting and formatting +4. Submit pull request +5. Code review and merge + +### Commit Messages +Use conventional commits format: +``` +feat: add new feature +fix: resolve issue +docs: update documentation +style: code formatting +refactor: code restructuring +test: add tests +chore: maintenance tasks +``` + +## 🤝 Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests for new functionality +5. Ensure all tests pass +6. Submit a pull request + +## 📄 License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. + +## 🙏 Acknowledgments + +- Apple Human Interface Guidelines for design inspiration +- FastAPI team for the excellent framework +- Next.js team for the amazing React framework +- OpenTelemetry community for observability tools + +## 📞 Support + +For support and questions: +- Create an issue on GitHub +- Check the documentation +- Contact the development team + +--- + +**Built with ❤️ using modern web technologies** \ No newline at end of file diff --git a/backend/.claude/settings.local.json b/backend/.claude/settings.local.json index 9e2c6c8..b0c5f58 100644 --- a/backend/.claude/settings.local.json +++ b/backend/.claude/settings.local.json @@ -1,7 +1,14 @@ { "permissions": { "allow": [ - "Bash(python:*)" + "Bash(python:*)", + "Bash(docker-compose:*)", + "Bash(docker swarm init:*)", + "Bash(docker build:*)", + "Bash(docker:*)", + "Bash(timeout:*)", + "Read(//e/projects/kami/kami_apple_exchage/frontend/**)", + "Read(//e/projects/kami/kami_apple_exchage/**)" ], "deny": [], "ask": [] diff --git a/backend/.env.example b/backend/.env.example index 8f949d1..5a2b223 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -45,7 +45,6 @@ ALLOWED_HOSTS=["*"] CORS_ORIGINS=["*"] CORS_METHODS=["*"] CORS_HEADERS=["*"] - # 日志配置 LOG_LEVEL=INFO LOG_FORMAT=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/backend/.env.local b/backend/.env.local index 9d188f8..5fc2897 100644 --- a/backend/.env.local +++ b/backend/.env.local @@ -51,4 +51,11 @@ PLAYWRIGHT_SLOW_MO=100 # 开发工具 ENABLE_DOCS=true ENABLE_REDOC=true -ENABLE_OPENAPI=true \ No newline at end of file +ENABLE_OPENAPI=true + +ALLOWED_HOSTS=["*"] + +# CORS配置 +CORS_ORIGINS=["*"] +CORS_METHODS=["*"] +CORS_HEADERS=["*"] \ No newline at end of file diff --git a/backend/.env.production b/backend/.env.production index d27c8df..4c8d614 100644 --- a/backend/.env.production +++ b/backend/.env.production @@ -10,14 +10,14 @@ PORT=8000 WORKERS=4 # 数据库配置 -DATABASE_URL="postgresql+asyncpg://postgres:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}" +DATABASE_URL="postgresql+asyncpg://postgres:password@db:5432/apple_exchange" DATABASE_POOL_SIZE=20 DATABASE_MAX_OVERFLOW=30 DATABASE_POOL_TIMEOUT=30 DATABASE_POOL_RECYCLE=3600 # Redis配置 -REDIS_URL="redis://:${REDIS_PASSWORD}@${REDIS_HOST}:${REDIS_PORT}/${REDIS_DB}" +REDIS_URL="redis://redis:6379/0" REDIS_MAX_CONNECTIONS=50 REDIS_RETRY_ON_TIMEOUT=true REDIS_SOCKET_KEEPALIVE=true @@ -48,8 +48,13 @@ LOG_RETENTION="30 days" # 安全配置 JWT_SECRET_KEY="${JWT_SECRET_KEY}" ENCRYPTION_KEY="${ENCRYPTION_KEY}" -ALLOWED_HOSTS="*" -CORS_ORIGINS="*" + +ALLOWED_HOSTS=["*"] + +# CORS配置 +CORS_ORIGINS=["*"] +CORS_METHODS=["*"] +CORS_HEADERS=["*"] # Playwright配置 PLAYWRIGHT_HEADLESS=true diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..062a315 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.13-slim + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV DEBIAN_FRONTEND=noninteractive \ + UV_COMPILE_BYTECODE=1 \ + PATH="/app/.venv/bin:$PATH" \ + UV_LINK_MODE=copy \ + TZ=Asia/Shanghai \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +COPY --chmod=755 docker-entrypoint.sh ./ + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + uv sync --frozen --no-install-project && \ + chmod -R 755 /app/.venv/bin/* + +RUN apt-get update && apt-get install -y \ + gcc g++ curl \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /app/screenshots /app/logs /app/data + +COPY app ./app +COPY run.py ./ +COPY .env.production .env + +HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8000/api/v1/health/liveness || exit 1 + +EXPOSE 8000 + +ENTRYPOINT ["./docker-entrypoint.sh"] \ No newline at end of file diff --git a/backend/Dockerfile.test b/backend/Dockerfile.test new file mode 100644 index 0000000..871abc2 --- /dev/null +++ b/backend/Dockerfile.test @@ -0,0 +1,11 @@ +FROM python:3.13-slim + +WORKDIR /app + +COPY docker-entrypoint-simple.sh ./ +RUN chmod +x docker-entrypoint-simple.sh + +COPY docker-entrypoint.sh ./ +RUN chmod +x docker-entrypoint.sh + +CMD ["./docker-entrypoint-simple.sh"] \ No newline at end of file diff --git a/backend/Dockerfile.worker b/backend/Dockerfile.worker new file mode 100644 index 0000000..6fd4f8f --- /dev/null +++ b/backend/Dockerfile.worker @@ -0,0 +1,41 @@ +FROM mcr.microsoft.com/playwright/python:v1.55.0-jammy + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV DEBIAN_FRONTEND=noninteractive \ + UV_COMPILE_BYTECODE=1 \ + PATH="/app/.venv/bin:$PATH" \ + UV_LINK_MODE=copy \ + TZ=Asia/Shanghai \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PLAYWRIGHT_BROWSERS_PATH=/app/playwright-browsers + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + gcc g++ curl wget gnupg \ + && rm -rf /var/lib/apt/lists/* && apt-get clean && apt-get autoremove -y + +COPY pyproject.toml uv.lock ./ +COPY --chmod=755 docker-entrypoint.sh ./ + +RUN uv sync --frozen && \ + chmod -R 755 /app/.venv/bin/* + +RUN mkdir -p /app/screenshots /app/logs /app/data /app/playwright-browsers + +COPY app ./app +COPY .env.production .env +COPY test_gunicorn.py ./ +COPY run.py ./ + +RUN python -m playwright install chromium && \ + python -m playwright install-deps chromium + +HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ + CMD python -c "from app.core.celery_app import get_celery_app; app = get_celery_app(); print('Worker healthy')" || exit 1 + +EXPOSE 8000 + +ENTRYPOINT ["./docker-entrypoint.sh"] \ No newline at end of file diff --git a/backend/alembic/versions/002_add_batch_tasks.py b/backend/alembic/versions/002_add_batch_tasks.py deleted file mode 100644 index fc92454..0000000 --- a/backend/alembic/versions/002_add_batch_tasks.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Add batch tasks tables - -Revision ID: 002 -Revises: 001 -Create Date: 2024-08-24 22:00:00.000000 - -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "002" -down_revision = "001" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # Create batch_tasks table - op.create_table( - "batch_tasks", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column( - "name", sa.String(length=255), nullable=False, comment="批量任务名称" - ), - sa.Column("description", sa.Text(), nullable=True, comment="任务描述"), - sa.Column( - "status", - sa.Enum( - "PENDING", - "PROCESSING", - "COMPLETED", - "FAILED", - "CANCELLED", - name="batchtaskstatus", - ), - nullable=True, - comment="任务状态", - ), - sa.Column("total_count", sa.Integer(), nullable=True, comment="总任务数量"), - sa.Column("processed_count", sa.Integer(), nullable=True, comment="已处理数量"), - sa.Column("success_count", sa.Integer(), nullable=True, comment="成功数量"), - sa.Column("failed_count", sa.Integer(), nullable=True, comment="失败数量"), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - comment="创建时间", - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - comment="更新时间", - ), - sa.Column( - "started_at", - sa.DateTime(timezone=True), - nullable=True, - comment="开始处理时间", - ), - sa.Column( - "completed_at", - sa.DateTime(timezone=True), - nullable=True, - comment="完成时间", - ), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_batch_tasks_id"), "batch_tasks", ["id"], unique=False) - - # Create batch_task_items table - op.create_table( - "batch_task_items", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("batch_task_id", sa.Integer(), nullable=False, comment="批量任务ID"), - sa.Column( - "order_url", sa.String(length=1000), nullable=False, comment="订单链接" - ), - sa.Column( - "amount", - sa.Numeric(precision=10, scale=2), - nullable=False, - comment="订单金额", - ), - sa.Column( - "status", - sa.Enum( - "PENDING", - "PROCESSING", - "SUCCESS", - "FAILED", - "SKIPPED", - name="batchtaskitemstatus", - ), - nullable=True, - comment="任务项状态", - ), - sa.Column( - "order_number", sa.String(length=255), nullable=True, comment="生成的订单号" - ), - sa.Column( - "final_order_url", - sa.String(length=1000), - nullable=True, - comment="最终订单URL", - ), - sa.Column("failure_reason", sa.Text(), nullable=True, comment="失败原因"), - sa.Column("retry_count", sa.Integer(), nullable=True, comment="重试次数"), - sa.Column("max_retries", sa.Integer(), nullable=True, comment="最大重试次数"), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - comment="创建时间", - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - comment="更新时间", - ), - sa.Column( - "started_at", - sa.DateTime(timezone=True), - nullable=True, - comment="开始处理时间", - ), - sa.Column( - "completed_at", - sa.DateTime(timezone=True), - nullable=True, - comment="完成时间", - ), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["batch_task_id"], ["batch_tasks.id"], ondelete="CASCADE" - ), - ) - op.create_index( - op.f("ix_batch_task_items_id"), "batch_task_items", ["id"], unique=False - ) - op.create_index( - "ix_batch_task_items_batch_task_id", - "batch_task_items", - ["batch_task_id"], - unique=False, - ) - op.create_index( - "ix_batch_task_items_status", "batch_task_items", ["status"], unique=False - ) - - -def downgrade() -> None: - # Drop indexes - op.drop_index("ix_batch_task_items_status", table_name="batch_task_items") - op.drop_index("ix_batch_task_items_batch_task_id", table_name="batch_task_items") - op.drop_index(op.f("ix_batch_task_items_id"), table_name="batch_task_items") - op.drop_index(op.f("ix_batch_tasks_id"), table_name="batch_tasks") - - # Drop tables - op.drop_table("batch_task_items") - op.drop_table("batch_tasks") - - # Drop enums - op.execute("DROP TYPE IF EXISTS batchtaskitemstatus") - op.execute("DROP TYPE IF EXISTS batchtaskstatus") diff --git a/backend/alembic/versions/add_links_table.py b/backend/alembic/versions/add_links_table.py deleted file mode 100644 index 5afc66e..0000000 --- a/backend/alembic/versions/add_links_table.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Add links table - -Revision ID: add_links_table -Revises: -Create Date: 2025-01-27 16:00:00.000000 - -""" - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = "add_links_table" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # Create links table if it doesn't exist - op.create_table( - "links", - sa.Column("id", sa.String(length=32), nullable=False, comment="主键ID"), - sa.Column("created_at", sa.DateTime(), nullable=False, comment="创建时间"), - sa.Column("updated_at", sa.DateTime(), nullable=False, comment="更新时间"), - sa.Column("url", sa.String(length=255), nullable=False), - sa.Column("amount", sa.Float(), nullable=False), - sa.PrimaryKeyConstraint("id"), - ) - - # Create index on url for faster lookups - op.create_index("ix_links_url", "links", ["url"]) - - # Create index on amount for range queries - op.create_index("ix_links_amount", "links", ["amount"]) - - -def downgrade() -> None: - op.drop_index("ix_links_amount", table_name="links") - op.drop_index("ix_links_url", table_name="links") - op.drop_table("links") diff --git a/backend/app/core/celery_app.py b/backend/app/core/celery_app.py index ea1355a..9ef85c0 100644 --- a/backend/app/core/celery_app.py +++ b/backend/app/core/celery_app.py @@ -39,6 +39,13 @@ celery_app.conf.update( # 导入任务模块 celery_app.autodiscover_tasks(["app.tasks"]) +# 显式导入任务模块以确保任务被注册 +try: + from app.tasks import crawler_tasks + logger.info("成功导入 crawler_tasks 模块") +except ImportError as e: + logger.error(f"导入 crawler_tasks 模块失败: {e}") + # 在Worker启动时初始化Playwright @worker_process_init.connect diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 31c0d21..9cc81ef 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -59,13 +59,9 @@ class Settings(BaseSettings): # 安全配置 SECRET_KEY: str = Field(default="your-secret-key-here", description="应用密钥") + # TODO: Fix environment variable parsing for ALLOWED_HOSTS and CORS_ORIGINS ALLOWED_HOSTS: list[str] = Field(default=["*"], description="允许的主机列表") - - # CORS配置 - CORS_ORIGINS: list[str] = Field( - default=["http://localhost:3000", "http://127.0.0.1:3000"], - description="CORS允许的源", - ) + CORS_ORIGINS: list[str] = Field(default=["*"], description="CORS允许的源列表") CORS_METHODS: list[str] = Field( default=["GET", "POST", "PUT", "DELETE", "OPTIONS"], description="CORS允许的方法", diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py index 8d37365..bb1357c 100644 --- a/backend/app/tasks/__init__.py +++ b/backend/app/tasks/__init__.py @@ -4,4 +4,7 @@ Celery任务模块 from app.core.celery_app import celery_app -__all__ = ["celery_app"] +# 导入所有任务模块以确保任务被注册 +from . import crawler_tasks + +__all__ = ["celery_app", "crawler_tasks"] diff --git a/backend/app/tasks/crawler_tasks.py b/backend/app/tasks/crawler_tasks.py index ab44e38..4ab9fd2 100644 --- a/backend/app/tasks/crawler_tasks.py +++ b/backend/app/tasks/crawler_tasks.py @@ -5,7 +5,6 @@ """ import asyncio -import uuid from typing import Any from datetime import datetime @@ -22,8 +21,7 @@ from app.enums.task import OrderTaskStatus from app.repositories.order_repository import OrderRepository from app.models.orders import OrderResultStatus from app.services.link_service import LinksService -from app.services.playwright_service import create_apple_order_processor, \ - AppleOrderProcessor +from app.services.playwright_service import AppleOrderProcessor from app.services.user_data_service import UserDataService logger = get_logger(__name__) diff --git a/backend/deploy/Dockerfile b/backend/deploy/Dockerfile deleted file mode 100644 index baf7559..0000000 --- a/backend/deploy/Dockerfile +++ /dev/null @@ -1,98 +0,0 @@ -# 多阶段构建的FastAPI应用Docker镜像 -FROM python:3.13-slim as builder - -# 设置工作目录 -WORKDIR /app - -# 安装系统依赖和Playwright依赖 -RUN apt-get update && apt-get install -y \ - gcc \ - g++ \ - curl \ - wget \ - gnupg \ - && rm -rf /var/lib/apt/lists/* - -# 安装uv包管理器 -RUN pip install uv - -# 复制项目文件 -COPY pyproject.toml uv.lock ./ - -# 安装Python依赖 -RUN uv sync --frozen - -# 生产阶段 -FROM python:3.13-slim - -# 接收构建参数 -ARG SERVICE_TYPE=api - -# 设置环境变量 -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PATH="/app/.venv/bin:$PATH" \ - PLAYWRIGHT_BROWSERS_PATH=/app/playwright-browsers \ - SERVICE_TYPE=$SERVICE_TYPE - -# 创建非root用户 -RUN groupadd -r appuser && useradd -r -g appuser appuser - -# 设置工作目录 -WORKDIR /app - -# 安装运行时依赖和Playwright系统依赖 -RUN apt-get update && apt-get install -y \ - curl \ - wget \ - gnupg \ - libnss3 \ - libatk-bridge2.0-0 \ - libdrm2 \ - libxkbcommon0 \ - libgtk-3-0 \ - libgbm1 \ - libasound2 \ - && rm -rf /var/lib/apt/lists/* - -# 从构建阶段复制虚拟环境 -COPY --from=builder /app/.venv /app/.venv - -# 创建必要的目录 -RUN mkdir -p /app/screenshots /app/logs /app/data /app/playwright-browsers - -# 复制应用代码 -COPY app ./app -COPY .env.production .env -COPY test_gunicorn.py ./ -COPY run.py ./ - -# 条件安装Playwright浏览器 - 仅在worker服务中安装 -RUN if [ "$SERVICE_TYPE" = "worker" ]; then \ - /app/.venv/bin/playwright install chromium && \ - /app/.venv/bin/playwright install-deps chromium; \ - fi - -# 更改文件所有权 -RUN chown -R appuser:appuser /app - -# 切换到非root用户 -USER appuser - -# 健康检查 - 支持不同的服务类型 -HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ - CMD if [ "$SERVICE_TYPE" = "worker" ]; then \ - python -c "from app.core.celery_app import get_celery_app; app = get_celery_app(); print('Worker healthy')" || exit 1; \ - else \ - curl -f http://localhost:8000/api/v1/health/liveness || exit 1; \ - fi - -# 暴露端口 -EXPOSE 8000 - -# 复制启动脚本 -COPY docker-entrypoint.sh ./ -RUN chmod +x docker-entrypoint.sh - -# 启动命令 - 支持不同的服务类型 -ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/backend/deploy/helm/apple-exchange/Chart.yaml b/backend/deploy/helm/apple-exchange/Chart.yaml deleted file mode 100644 index 60a5faa..0000000 --- a/backend/deploy/helm/apple-exchange/Chart.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v2 -name: apple-exchange -description: A Helm chart for Apple Exchange application -type: application -version: 0.1.0 -appVersion: "2.0.0" -maintainers: - - name: DevOps Team - email: devops@example.com diff --git a/backend/deploy/helm/apple-exchange/README.md b/backend/deploy/helm/apple-exchange/README.md deleted file mode 100644 index 526008a..0000000 --- a/backend/deploy/helm/apple-exchange/README.md +++ /dev/null @@ -1,129 +0,0 @@ -# Apple Exchange Helm Chart - -This Helm chart deploys the Apple Exchange application on a Kubernetes cluster. - -## Prerequisites - -- Kubernetes 1.19+ -- Helm 3.2.0+ -- PV provisioner support in the underlying infrastructure -- Redis and PostgreSQL databases (can be deployed separately) - -## Installing the Chart - -To install the chart with the release name `apple-exchange`: - -```bash -helm install apple-exchange ./apple-exchange -``` - -## Configuration - -The following table lists the configurable parameters of the Apple Exchange chart and their default values. - -### Global Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `replicaCount.api` | Number of API replicas | `3` | -| `replicaCount.worker` | Number of worker replicas | `4` | -| `replicaCount.beat` | Number of beat replicas | `1` | -| `replicaCount.flower` | Number of flower replicas | `1` | -| `image.repository` | Image repository | `apple-exchange-api` | -| `image.tag` | Image tag | `latest` | -| `image.pullPolicy` | Image pull policy | `Always` | - -### Service Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `service.api.type` | API service type | `ClusterIP` | -| `service.api.port` | API service port | `8000` | -| `service.flower.type` | Flower service type | `ClusterIP` | -| `service.flower.port` | Flower service port | `5555` | - -### Ingress Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `ingress.enabled` | Enable ingress | `true` | -| `ingress.className` | Ingress class name | `nginx` | -| `ingress.annotations` | Ingress annotations | `{}` | -| `ingress.hosts` | Ingress hosts | `[]` | -| `ingress.tls` | Ingress TLS configuration | `[]` | - -### Resource Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `resources.api.limits.cpu` | API CPU limit | `1000m` | -| `resources.api.limits.memory` | API memory limit | `1Gi` | -| `resources.api.requests.cpu` | API CPU request | `500m` | -| `resources.api.requests.memory` | API memory request | `512Mi` | -| `resources.worker.limits.cpu` | Worker CPU limit | `1500m` | -| `resources.worker.limits.memory` | Worker memory limit | `2Gi` | -| `resources.worker.requests.cpu` | Worker CPU request | `500m` | -| `resources.worker.requests.memory` | Worker memory request | `1Gi` | - -### Autoscaling Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `autoscaling.api.enabled` | Enable API autoscaling | `true` | -| `autoscaling.api.minReplicas` | API minimum replicas | `2` | -| `autoscaling.api.maxReplicas` | API maximum replicas | `10` | -| `autoscaling.api.targetCPUUtilizationPercentage` | API target CPU utilization | `80` | -| `autoscaling.worker.enabled` | Enable worker autoscaling | `true` | -| `autoscaling.worker.minReplicas` | Worker minimum replicas | `2` | -| `autoscaling.worker.maxReplicas` | Worker maximum replicas | `20` | -| `autoscaling.worker.targetCPUUtilizationPercentage` | Worker target CPU utilization | `70` | - -### Persistence Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `persistence.logs.enabled` | Enable logs persistence | `true` | -| `persistence.logs.storageClass` | Logs storage class | `""` | -| `persistence.logs.accessMode` | Logs access mode | `ReadWriteMany` | -| `persistence.logs.size` | Logs storage size | `10Gi` | -| `persistence.screenshots.enabled` | Enable screenshots persistence | `true` | -| `persistence.screenshots.storageClass` | Screenshots storage class | `""` | -| `persistence.screenshots.accessMode` | Screenshots access mode | `ReadWriteMany` | -| `persistence.screenshots.size` | Screenshots storage size | `20Gi` | - -### Database Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `database.host` | Database host | `postgres-postgresql.database` | -| `database.port` | Database port | `5432` | -| `database.name` | Database name | `apple_exchange` | -| `database.user` | Database user | `postgres` | - -### Redis Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `redis.host` | Redis host | `redis-master.database` | -| `redis.port` | Redis port | `6379` | -| `redis.db` | Redis database | `0` | - -### Environment Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `env.ENVIRONMENT` | Environment | `production` | -| `env.CELERY_CONCURRENCY` | Celery concurrency | `100` | -| `env.CELERY_MAX_TASKS_PER_CHILD` | Celery max tasks per child | `1000` | -| `env.CELERY_PREFETCH_MULTIPLIER` | Celery prefetch multiplier | `1` | -| `env.WORKERS` | Number of workers | `4` | -| `env.SCREENSHOT_DIR` | Screenshot directory | `/app/screenshots` | -| `env.LOG_DIR` | Log directory | `/app/logs` | -| `env.PLAYWRIGHT_BROWSERS_PATH` | Playwright browsers path | `/app/playwright-browsers` | - -## Upgrading the Chart - -To upgrade the chart: - -```bash -helm upgrade apple-exchange ./apple-exchange diff --git a/backend/deploy/helm/apple-exchange/templates/NOTES.txt b/backend/deploy/helm/apple-exchange/templates/NOTES.txt deleted file mode 100644 index 756e3f2..0000000 --- a/backend/deploy/helm/apple-exchange/templates/NOTES.txt +++ /dev/null @@ -1,27 +0,0 @@ -Thank you for installing {{ .Chart.Name }}. - -Your release is named {{ .Release.Name }}. - -To learn more about the release, try: - - $ helm status {{ .Release.Name }} - $ helm get all {{ .Release.Name }} - -{{- if eq .Values.service.api.type "NodePort" }} -You can access the application at: - - API: http://:{{ .Values.service.api.nodePort }} - - Flower: http://:{{ .Values.service.flower.nodePort }} - -Replace with the IP address of any node in your Kubernetes cluster. -{{- else }} -To access the application, you need to forward the service ports: - - $ kubectl port-forward svc/{{ include "apple-exchange.fullname" . }}-api 8000:8000 -n {{ .Release.Namespace }} - $ kubectl port-forward svc/{{ include "apple-exchange.fullname" . }}-flower 5555:5555 -n {{ .Release.Namespace }} - -Then you can access: - - API: http://localhost:8000 - - Flower: http://localhost:5555 -{{- end }} - -For more information, please refer to the documentation. diff --git a/backend/deploy/helm/apple-exchange/templates/_helpers.tpl b/backend/deploy/helm/apple-exchange/templates/_helpers.tpl deleted file mode 100644 index 1b74e64..0000000 --- a/backend/deploy/helm/apple-exchange/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "apple-exchange.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "apple-exchange.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "apple-exchange.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "apple-exchange.labels" -}} -helm.sh/chart: {{ include "apple-exchange.chart" . }} -{{ include "apple-exchange.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "apple-exchange.selectorLabels" -}} -app.kubernetes.io/name: {{ include "apple-exchange.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "apple-exchange.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "apple-exchange.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/backend/deploy/helm/apple-exchange/templates/configmap.yaml b/backend/deploy/helm/apple-exchange/templates/configmap.yaml deleted file mode 100644 index 9453fba..0000000 --- a/backend/deploy/helm/apple-exchange/templates/configmap.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "apple-exchange.fullname" . }}-config - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} -data: - ENVIRONMENT: {{ .Values.env.ENVIRONMENT | quote }} - DATABASE_HOST: {{ .Values.database.host | quote }} - DATABASE_PORT: {{ .Values.database.port | quote }} - DATABASE_NAME: {{ .Values.database.name | quote }} - DATABASE_USER: {{ .Values.database.user | quote }} - REDIS_HOST: {{ .Values.redis.host | quote }} - REDIS_PORT: {{ .Values.redis.port | quote }} - REDIS_DB: {{ .Values.redis.db | quote }} - CELERY_CONCURRENCY: {{ .Values.env.CELERY_CONCURRENCY | quote }} - CELERY_MAX_TASKS_PER_CHILD: {{ .Values.env.CELERY_MAX_TASKS_PER_CHILD | quote }} - CELERY_PREFETCH_MULTIPLIER: {{ .Values.env.CELERY_PREFETCH_MULTIPLIER | quote }} - WORKERS: {{ .Values.env.WORKERS | quote }} - SCREENSHOT_DIR: {{ .Values.env.SCREENSHOT_DIR | quote }} - LOG_DIR: {{ .Values.env.LOG_DIR | quote }} - PLAYWRIGHT_BROWSERS_PATH: {{ .Values.env.PLAYWRIGHT_BROWSERS_PATH | quote }} \ No newline at end of file diff --git a/backend/deploy/helm/apple-exchange/templates/deployment-api.yaml b/backend/deploy/helm/apple-exchange/templates/deployment-api.yaml deleted file mode 100644 index 7c624b9..0000000 --- a/backend/deploy/helm/apple-exchange/templates/deployment-api.yaml +++ /dev/null @@ -1,127 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "apple-exchange.fullname" . }}-api - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - component: api -spec: - {{- if not .Values.autoscaling.api.enabled }} - replicas: {{ .Values.replicaCount.api }} - {{- end }} - selector: - matchLabels: - {{- include "apple-exchange.selectorLabels" . | nindent 6 }} - component: api - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "apple-exchange.selectorLabels" . | nindent 8 }} - component: api - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "apple-exchange.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }}-api - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - env: - - name: SERVICE_TYPE - value: "api" - - name: DATABASE_URL - value: "postgresql+asyncpg://{{ .Values.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.name }}" - - name: REDIS_URL - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/{{ .Values.redis.db }}" - - name: CELERY_BROKER_URL - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/0" - - name: CELERY_RESULT_BACKEND - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/1" - - name: SCREENSHOT_DIR - value: {{ .Values.env.SCREENSHOT_DIR }} - - name: LOG_DIR - value: {{ .Values.env.LOG_DIR }} - - name: WORKERS - value: "{{ .Values.env.WORKERS }}" - - name: ENVIRONMENT - value: {{ .Values.env.ENVIRONMENT }} - envFrom: - - secretRef: - name: {{ include "apple-exchange.fullname" . }}-secret - - configMapRef: - name: {{ include "apple-exchange.fullname" . }}-config - livenessProbe: - httpGet: - path: /api/v1/health/liveness - port: http - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /api/v1/health/readiness - port: http - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - startupProbe: - httpGet: - path: /api/v1/health/startup - port: http - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 30 - resources: - {{- toYaml .Values.resources.api | nindent 12 }} - volumeMounts: - - name: logs - mountPath: {{ .Values.env.LOG_DIR }} - - name: screenshots - mountPath: {{ .Values.env.SCREENSHOT_DIR }} - - name: shared - mountPath: /app/shared - volumes: - - name: logs - persistentVolumeClaim: - claimName: {{ include "apple-exchange.fullname" . }}-logs - - name: screenshots - persistentVolumeClaim: - claimName: {{ include "apple-exchange.fullname" . }}-screenshots - - name: shared - persistentVolumeClaim: - claimName: {{ include "apple-exchange.fullname" . }}-shared - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/backend/deploy/helm/apple-exchange/templates/deployment-beat.yaml b/backend/deploy/helm/apple-exchange/templates/deployment-beat.yaml deleted file mode 100644 index 479a5cc..0000000 --- a/backend/deploy/helm/apple-exchange/templates/deployment-beat.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "apple-exchange.fullname" . }}-beat - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - component: beat -spec: - replicas: 1 # Beat should always have exactly one replica - selector: - matchLabels: - {{- include "apple-exchange.selectorLabels" . | nindent 6 }} - component: beat - strategy: - type: Recreate # Ensure only one instance runs at a time - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "apple-exchange.selectorLabels" . | nindent 8 }} - component: beat - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "apple-exchange.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }}-beat - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: SERVICE_TYPE - value: "beat" - - name: DATABASE_URL - value: "postgresql+asyncpg://{{ .Values.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.name }}" - - name: REDIS_URL - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/{{ .Values.redis.db }}" - - name: CELERY_BROKER_URL - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/0" - - name: CELERY_RESULT_BACKEND - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/1" - - name: ENVIRONMENT - value: {{ .Values.env.ENVIRONMENT }} - - name: LOG_DIR - value: {{ .Values.env.LOG_DIR }} - envFrom: - - secretRef: - name: {{ include "apple-exchange.fullname" . }}-secret - - configMapRef: - name: {{ include "apple-exchange.fullname" . }}-config - resources: - {{- toYaml .Values.resources.beat | nindent 12 }} - volumeMounts: - - name: logs - mountPath: {{ .Values.env.LOG_DIR }} - - name: beat-schedule - mountPath: /app/data - volumes: - - name: logs - persistentVolumeClaim: - claimName: {{ include "apple-exchange.fullname" . }}-logs - - name: beat-schedule - emptyDir: {} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/backend/deploy/helm/apple-exchange/templates/deployment-flower.yaml b/backend/deploy/helm/apple-exchange/templates/deployment-flower.yaml deleted file mode 100644 index 1b6d93f..0000000 --- a/backend/deploy/helm/apple-exchange/templates/deployment-flower.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "apple-exchange.fullname" . }}-flower - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - component: flower -spec: - replicas: {{ .Values.replicaCount.flower }} - selector: - matchLabels: - {{- include "apple-exchange.selectorLabels" . | nindent 6 }} - component: flower - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "apple-exchange.selectorLabels" . | nindent 8 }} - component: flower - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "apple-exchange.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }}-flower - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: flower - containerPort: 5555 - protocol: TCP - env: - - name: SERVICE_TYPE - value: "flower" - - name: REDIS_URL - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/{{ .Values.redis.db }}" - - name: CELERY_BROKER_URL - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/0" - - name: ENVIRONMENT - value: {{ .Values.env.ENVIRONMENT }} - envFrom: - - secretRef: - name: {{ include "apple-exchange.fullname" . }}-secret - - configMapRef: - name: {{ include "apple-exchange.fullname" . }}-config - livenessProbe: - httpGet: - path: / - port: flower - initialDelaySeconds: 30 - periodSeconds: 30 - readinessProbe: - httpGet: - path: / - port: flower - initialDelaySeconds: 10 - periodSeconds: 10 - resources: - {{- toYaml .Values.resources.flower | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/backend/deploy/helm/apple-exchange/templates/deployment-worker.yaml b/backend/deploy/helm/apple-exchange/templates/deployment-worker.yaml deleted file mode 100644 index 08d68ba..0000000 --- a/backend/deploy/helm/apple-exchange/templates/deployment-worker.yaml +++ /dev/null @@ -1,130 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "apple-exchange.fullname" . }}-worker - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - component: worker -spec: - {{- if not .Values.autoscaling.worker.enabled }} - replicas: {{ .Values.replicaCount.worker }} - {{- end }} - selector: - matchLabels: - {{- include "apple-exchange.selectorLabels" . | nindent 6 }} - component: worker - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 2 - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "apple-exchange.selectorLabels" . | nindent 8 }} - component: worker - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "apple-exchange.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }}-worker - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: SERVICE_TYPE - value: "worker" - - name: DATABASE_URL - value: "postgresql+asyncpg://{{ .Values.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.name }}" - - name: REDIS_URL - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/{{ .Values.redis.db }}" - - name: CELERY_BROKER_URL - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/0" - - name: CELERY_RESULT_BACKEND - value: "redis://{{ .Values.redis.host }}:{{ .Values.redis.port }}/1" - - name: CELERY_CONCURRENCY - value: "{{ .Values.env.CELERY_CONCURRENCY }}" - - name: CELERY_MAX_TASKS_PER_CHILD - value: "{{ .Values.env.CELERY_MAX_TASKS_PER_CHILD }}" - - name: CELERY_PREFETCH_MULTIPLIER - value: "{{ .Values.env.CELERY_PREFETCH_MULTIPLIER }}" - - name: SCREENSHOT_DIR - value: {{ .Values.env.SCREENSHOT_DIR }} - - name: LOG_DIR - value: {{ .Values.env.LOG_DIR }} - - name: PLAYWRIGHT_BROWSERS_PATH - value: {{ .Values.env.PLAYWRIGHT_BROWSERS_PATH }} - - name: ENVIRONMENT - value: {{ .Values.env.ENVIRONMENT }} - envFrom: - - secretRef: - name: {{ include "apple-exchange.fullname" . }}-secret - - configMapRef: - name: {{ include "apple-exchange.fullname" . }}-config - livenessProbe: - exec: - command: - - python - - -c - - "from app.core.celery_app import get_celery_app; app = get_celery_app(); print('Worker healthy')" - initialDelaySeconds: 60 - periodSeconds: 60 - timeoutSeconds: 30 - failureThreshold: 3 - readinessProbe: - exec: - command: - - python - - -c - - "from app.core.celery_app import get_celery_app; app = get_celery_app(); print('Worker ready')" - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - resources: - {{- toYaml .Values.resources.worker | nindent 12 }} - volumeMounts: - - name: logs - mountPath: {{ .Values.env.LOG_DIR }} - - name: screenshots - mountPath: {{ .Values.env.SCREENSHOT_DIR }} - - name: shared - mountPath: /app/shared - - name: playwright-browsers - mountPath: {{ .Values.env.PLAYWRIGHT_BROWSERS_PATH }} - volumes: - - name: logs - persistentVolumeClaim: - claimName: {{ include "apple-exchange.fullname" . }}-logs - - name: screenshots - persistentVolumeClaim: - claimName: {{ include "apple-exchange.fullname" . }}-screenshots - - name: shared - persistentVolumeClaim: - claimName: {{ include "apple-exchange.fullname" . }}-shared - - name: playwright-browsers - emptyDir: - sizeLimit: 2Gi - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/backend/deploy/helm/apple-exchange/templates/hpa.yaml b/backend/deploy/helm/apple-exchange/templates/hpa.yaml deleted file mode 100644 index 3fd0e57..0000000 --- a/backend/deploy/helm/apple-exchange/templates/hpa.yaml +++ /dev/null @@ -1,67 +0,0 @@ -{{- if .Values.autoscaling.api.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "apple-exchange.fullname" . }}-api - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - component: api -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "apple-exchange.fullname" . }}-api - minReplicas: {{ .Values.autoscaling.api.minReplicas }} - maxReplicas: {{ .Values.autoscaling.api.maxReplicas }} - metrics: - {{- if .Values.autoscaling.api.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.api.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.api.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.api.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} ---- -{{- if .Values.autoscaling.worker.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "apple-exchange.fullname" . }}-worker - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - component: worker -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "apple-exchange.fullname" . }}-worker - minReplicas: {{ .Values.autoscaling.worker.minReplicas }} - maxReplicas: {{ .Values.autoscaling.worker.maxReplicas }} - metrics: - {{- if .Values.autoscaling.worker.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.worker.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.worker.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.worker.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/backend/deploy/helm/apple-exchange/templates/ingress.yaml b/backend/deploy/helm/apple-exchange/templates/ingress.yaml deleted file mode 100644 index 7211da5..0000000 --- a/backend/deploy/helm/apple-exchange/templates/ingress.yaml +++ /dev/null @@ -1,52 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "apple-exchange.fullname" . -}} -{{- $svcPort := .Values.service.api.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.className }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - pathType: {{ .pathType }} - backend: - service: - name: {{ $fullName }}-{{ .service }} - port: - {{- if eq .service "api" }} - number: {{ $.Values.service.api.port }} - {{- else if eq .service "flower" }} - number: {{ $.Values.service.flower.port }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/backend/deploy/helm/apple-exchange/templates/pvc.yaml b/backend/deploy/helm/apple-exchange/templates/pvc.yaml deleted file mode 100644 index 37d5518..0000000 --- a/backend/deploy/helm/apple-exchange/templates/pvc.yaml +++ /dev/null @@ -1,53 +0,0 @@ -{{- if .Values.persistence.logs.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "apple-exchange.fullname" . }}-logs - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} -spec: - accessModes: - - {{ .Values.persistence.logs.accessMode }} - {{- if .Values.persistence.logs.storageClass }} - storageClassName: {{ .Values.persistence.logs.storageClass }} - {{- end }} - resources: - requests: - storage: {{ .Values.persistence.logs.size }} -{{- end }} ---- -{{- if .Values.persistence.screenshots.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "apple-exchange.fullname" . }}-screenshots - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} -spec: - accessModes: - - {{ .Values.persistence.screenshots.accessMode }} - {{- if .Values.persistence.screenshots.storageClass }} - storageClassName: {{ .Values.persistence.screenshots.storageClass }} - {{- end }} - resources: - requests: - storage: {{ .Values.persistence.screenshots.size }} -{{- end }} ---- -{{- if .Values.persistence.shared.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "apple-exchange.fullname" . }}-shared - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} -spec: - accessModes: - - {{ .Values.persistence.shared.accessMode }} - {{- if .Values.persistence.shared.storageClass }} - storageClassName: {{ .Values.persistence.shared.storageClass }} - {{- end }} - resources: - requests: - storage: {{ .Values.persistence.shared.size }} -{{- end }} diff --git a/backend/deploy/helm/apple-exchange/templates/secret.yaml b/backend/deploy/helm/apple-exchange/templates/secret.yaml deleted file mode 100644 index 27733f3..0000000 --- a/backend/deploy/helm/apple-exchange/templates/secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "apple-exchange.fullname" . }}-secret - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} -type: Opaque -data: - # These are placeholder values that should be overridden in production - DATABASE_PASSWORD: {{ randAlphaNum 16 | b64enc | quote }} - # Add other secrets as needed diff --git a/backend/deploy/helm/apple-exchange/templates/service.yaml b/backend/deploy/helm/apple-exchange/templates/service.yaml deleted file mode 100644 index 49ddfc2..0000000 --- a/backend/deploy/helm/apple-exchange/templates/service.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "apple-exchange.fullname" . }}-api - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - component: api -spec: - type: {{ .Values.service.api.type }} - ports: - - port: {{ .Values.service.api.port }} - targetPort: http - protocol: TCP - name: http - {{- if and (eq .Values.service.api.type "NodePort") .Values.service.api.nodePort }} - nodePort: {{ .Values.service.api.nodePort }} - {{- end }} - selector: - {{- include "apple-exchange.selectorLabels" . | nindent 4 }} - component: api ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "apple-exchange.fullname" . }}-flower - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - component: flower -spec: - type: {{ .Values.service.flower.type }} - ports: - - port: {{ .Values.service.flower.port }} - targetPort: flower - protocol: TCP - name: flower - {{- if and (eq .Values.service.flower.type "NodePort") .Values.service.flower.nodePort }} - nodePort: {{ .Values.service.flower.nodePort }} - {{- end }} - selector: - {{- include "apple-exchange.selectorLabels" . | nindent 4 }} - component: flower diff --git a/backend/deploy/helm/apple-exchange/templates/serviceaccount.yaml b/backend/deploy/helm/apple-exchange/templates/serviceaccount.yaml deleted file mode 100644 index 945de39..0000000 --- a/backend/deploy/helm/apple-exchange/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "apple-exchange.serviceAccountName" . }} - labels: - {{- include "apple-exchange.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/backend/deploy/helm/apple-exchange/values.yaml b/backend/deploy/helm/apple-exchange/values.yaml deleted file mode 100644 index 0a6356e..0000000 --- a/backend/deploy/helm/apple-exchange/values.yaml +++ /dev/null @@ -1,132 +0,0 @@ -# Default values for apple-exchange -# This is a YAML-formatted file. - -replicaCount: - api: 3 - worker: 4 - beat: 1 - flower: 1 - -image: - repository: apple-exchange-api - tag: latest - pullPolicy: Always - -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - create: true - annotations: {} - name: "" - -podAnnotations: {} -podSecurityContext: {} - -securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - -service: - api: - type: NodePort - port: 8000 - nodePort: 30080 - flower: - type: NodePort - port: 5555 - nodePort: 30555 - -ingress: - enabled: false - -resources: - api: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 500m - memory: 512Mi - worker: - limits: - cpu: 1500m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - beat: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 250m - memory: 256Mi - flower: - limits: - cpu: 200m - memory: 256Mi - requests: - cpu: 100m - memory: 128Mi - -autoscaling: - api: - enabled: true - minReplicas: 2 - maxReplicas: 10 - targetCPUUtilizationPercentage: 80 - targetMemoryUtilizationPercentage: 80 - worker: - enabled: true - minReplicas: 2 - maxReplicas: 20 - targetCPUUtilizationPercentage: 70 - targetMemoryUtilizationPercentage: 70 - -nodeSelector: {} -tolerations: [] -affinity: {} - -persistence: - logs: - enabled: true - storageClass: "" - accessMode: ReadWriteMany - size: 10Gi - screenshots: - enabled: true - storageClass: "" - accessMode: ReadWriteMany - size: 20Gi - shared: - enabled: true - storageClass: "" - accessMode: ReadWriteMany - size: 5Gi - -# Database configuration -database: - host: "postgres-postgresql.database" - port: 5432 - name: "apple_exchange" - user: "postgres" - # password should be set in a secret - -# Redis configuration -redis: - host: "redis-master.database" - port: 6379 - db: 0 - -# Environment configuration -env: - ENVIRONMENT: "production" - CELERY_CONCURRENCY: "100" - CELERY_MAX_TASKS_PER_CHILD: "1000" - CELERY_PREFETCH_MULTIPLIER: "1" - WORKERS: "4" - SCREENSHOT_DIR: "/app/screenshots" - LOG_DIR: "/app/logs" - PLAYWRIGHT_BROWSERS_PATH: "/app/playwright-browsers" diff --git a/backend/deploy/k8s/configmap.yaml b/backend/deploy/k8s/configmap.yaml deleted file mode 100644 index 6f00662..0000000 --- a/backend/deploy/k8s/configmap.yaml +++ /dev/null @@ -1,87 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: apple-exchange-config - namespace: apple-exchange -data: - # 应用配置 - APP_NAME: "Apple Gift Card Exchange" - APP_VERSION: "2.0.0" - ENVIRONMENT: "production" - HOST: "0.0.0.0" - PORT: "8000" - WORKERS: "4" - - # 数据库配置 - DATABASE_HOST: "postgres-service" - DATABASE_PORT: "5432" - DATABASE_NAME: "apple_exchange" - DATABASE_USER: "postgres" - - # Redis配置 - REDIS_HOST: "redis-service" - REDIS_PORT: "6379" - REDIS_DB: "0" - - # Celery配置 - CELERY_BROKER_URL: "redis://redis-service:6379/0" - CELERY_RESULT_BACKEND: "redis://redis-service:6379/1" - CELERY_TASK_SERIALIZER: "json" - CELERY_RESULT_SERIALIZER: "json" - CELERY_ACCEPT_CONTENT: "json" - CELERY_TIMEZONE: "Asia/Shanghai" - CELERY_ENABLE_UTC: "true" - CELERY_TASK_TRACK_STARTED: "true" - CELERY_TASK_TIME_LIMIT: "3600" - CELERY_TASK_SOFT_TIME_LIMIT: "3300" - CELERY_WORKER_PREFETCH_MULTIPLIER: "1" - CELERY_WORKER_MAX_TASKS_PER_CHILD: "1000" - CELERY_WORKER_DISABLE_RATE_LIMITS: "true" - - # 分布式锁配置 - DISTRIBUTED_LOCK_TIMEOUT: "300" - DISTRIBUTED_LOCK_EXTEND_INTERVAL: "60" - DISTRIBUTED_LOCK_RETRY_DELAY: "1" - DISTRIBUTED_LOCK_MAX_RETRIES: "3" - - # 文件存储配置 - SCREENSHOT_DIR: "/app/screenshots" - LOG_DIR: "/app/logs" - SHARED_DIR: "/app/shared" - FILE_CLEANUP_DAYS: "7" - MAX_SCREENSHOT_SIZE: "10485760" # 10MB - - # Playwright配置 - PLAYWRIGHT_BROWSERS_PATH: "/app/playwright-browsers" - PLAYWRIGHT_TIMEOUT: "30000" - PLAYWRIGHT_NAVIGATION_TIMEOUT: "30000" - PLAYWRIGHT_HEADLESS: "true" - PLAYWRIGHT_SLOW_MO: "0" - - # OpenTelemetry配置 - OTEL_SERVICE_NAME: "apple-exchange-api" - OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger-collector:14250" - OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" - - # 线程池配置 (已弃用，使用Celery替代) - DEFAULT_THREAD_POOL_SIZE: "10" - MAX_THREAD_POOL_SIZE: "50" - - # 礼品卡配置 - DEFAULT_GIFT_CARD_TIMEOUT: "300" - MAX_GIFT_CARD_TIMEOUT: "3600" - - # 日志配置 - LOG_LEVEL: "INFO" - LOG_FORMAT: "json" - LOG_ROTATION: "1 day" - LOG_RETENTION: "30 days" - - # 健康检查配置 - HEALTH_CHECK_TIMEOUT: "10" - HEALTH_CHECK_INTERVAL: "30" - - # 监控配置 - METRICS_ENABLED: "true" - METRICS_PORT: "9090" - PROMETHEUS_MULTIPROC_DIR: "/tmp/prometheus_multiproc" diff --git a/backend/deploy/k8s/deployment.yaml b/backend/deploy/k8s/deployment.yaml deleted file mode 100644 index e08192c..0000000 --- a/backend/deploy/k8s/deployment.yaml +++ /dev/null @@ -1,349 +0,0 @@ -# API服务部署 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: apple-exchange-api - namespace: apple-exchange - labels: - app: apple-exchange-api - component: api - version: v2.0.0 -spec: - replicas: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - selector: - matchLabels: - app: apple-exchange-api - component: api - template: - metadata: - labels: - app: apple-exchange-api - component: api - version: v2.0.0 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8000" - prometheus.io/path: "/api/v1/health/metrics" - spec: - containers: - - name: api - image: apple-exchange-api:latest - imagePullPolicy: Always - ports: - - containerPort: 8000 - name: http - env: - - name: SERVICE_TYPE - value: "api" - - name: DATABASE_URL - value: "postgresql+asyncpg://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(DATABASE_NAME)" - - name: REDIS_URL - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/$(REDIS_DB)" - - name: CELERY_BROKER_URL - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/0" - - name: CELERY_RESULT_BACKEND - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/1" - - name: SCREENSHOT_DIR - value: "/app/screenshots" - - name: LOG_DIR - value: "/app/logs" - envFrom: - - configMapRef: - name: apple-exchange-config - - secretRef: - name: apple-exchange-secret - resources: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "1Gi" - cpu: "1000m" - livenessProbe: - httpGet: - path: /api/v1/health/liveness - port: 8000 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /api/v1/health/readiness - port: 8000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - startupProbe: - httpGet: - path: /api/v1/health/startup - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 30 - volumeMounts: - - name: logs - mountPath: /app/logs - - name: screenshots - mountPath: /app/screenshots - - name: shared - mountPath: /app/shared - volumes: - - name: logs - persistentVolumeClaim: - claimName: apple-exchange-logs-pvc - - name: screenshots - persistentVolumeClaim: - claimName: apple-exchange-screenshots-pvc - - name: shared - persistentVolumeClaim: - claimName: apple-exchange-shared-pvc - restartPolicy: Always ---- -# Celery Worker部署 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: apple-exchange-worker - namespace: apple-exchange - labels: - app: apple-exchange-worker - component: worker - version: v2.0.0 -spec: - replicas: 4 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 2 - selector: - matchLabels: - app: apple-exchange-worker - component: worker - template: - metadata: - labels: - app: apple-exchange-worker - component: worker - version: v2.0.0 - annotations: - prometheus.io/scrape: "false" - spec: - containers: - - name: worker - image: apple-exchange-api:latest - imagePullPolicy: Always - env: - - name: SERVICE_TYPE - value: "worker" - - name: DATABASE_URL - value: "postgresql+asyncpg://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(DATABASE_NAME)" - - name: REDIS_URL - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/$(REDIS_DB)" - - name: CELERY_BROKER_URL - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/0" - - name: CELERY_RESULT_BACKEND - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/1" - - name: CELERY_CONCURRENCY - value: "2" - - name: CELERY_MAX_TASKS_PER_CHILD - value: "1000" - - name: CELERY_PREFETCH_MULTIPLIER - value: "1" - - name: SCREENSHOT_DIR - value: "/app/screenshots" - - name: LOG_DIR - value: "/app/logs" - - name: PLAYWRIGHT_BROWSERS_PATH - value: "/app/playwright-browsers" - envFrom: - - configMapRef: - name: apple-exchange-config - - secretRef: - name: apple-exchange-secret - resources: - requests: - memory: "1Gi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "1500m" - livenessProbe: - exec: - command: - - python - - -c - - "from app.core.celery_app import get_celery_app; app = get_celery_app(); print('Worker healthy')" - initialDelaySeconds: 60 - periodSeconds: 60 - timeoutSeconds: 30 - failureThreshold: 3 - readinessProbe: - exec: - command: - - python - - -c - - "from app.core.celery_app import get_celery_app; app = get_celery_app(); print('Worker ready')" - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - volumeMounts: - - name: logs - mountPath: /app/logs - - name: screenshots - mountPath: /app/screenshots - - name: shared - mountPath: /app/shared - - name: playwright-browsers - mountPath: /app/playwright-browsers - volumes: - - name: logs - persistentVolumeClaim: - claimName: apple-exchange-logs-pvc - - name: screenshots - persistentVolumeClaim: - claimName: apple-exchange-screenshots-pvc - - name: shared - persistentVolumeClaim: - claimName: apple-exchange-shared-pvc - - name: playwright-browsers - emptyDir: - sizeLimit: 2Gi - restartPolicy: Always ---- -# Celery Beat调度器部署 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: apple-exchange-beat - namespace: apple-exchange - labels: - app: apple-exchange-beat - component: beat - version: v2.0.0 -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: apple-exchange-beat - component: beat - template: - metadata: - labels: - app: apple-exchange-beat - component: beat - version: v2.0.0 - spec: - containers: - - name: beat - image: apple-exchange-api:latest - imagePullPolicy: Always - env: - - name: SERVICE_TYPE - value: "beat" - - name: DATABASE_URL - value: "postgresql+asyncpg://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(DATABASE_NAME)" - - name: REDIS_URL - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/$(REDIS_DB)" - - name: CELERY_BROKER_URL - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/0" - - name: CELERY_RESULT_BACKEND - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/1" - envFrom: - - configMapRef: - name: apple-exchange-config - - secretRef: - name: apple-exchange-secret - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "512Mi" - cpu: "500m" - volumeMounts: - - name: logs - mountPath: /app/logs - - name: beat-schedule - mountPath: /app/data - volumes: - - name: logs - persistentVolumeClaim: - claimName: apple-exchange-logs-pvc - - name: beat-schedule - emptyDir: {} - restartPolicy: Always ---- -# Celery Flower监控部署 -apiVersion: apps/v1 -kind: Deployment -metadata: - name: apple-exchange-flower - namespace: apple-exchange - labels: - app: apple-exchange-flower - component: flower - version: v2.0.0 -spec: - replicas: 1 - selector: - matchLabels: - app: apple-exchange-flower - component: flower - template: - metadata: - labels: - app: apple-exchange-flower - component: flower - version: v2.0.0 - spec: - containers: - - name: flower - image: apple-exchange-api:latest - imagePullPolicy: Always - ports: - - containerPort: 5555 - name: flower - env: - - name: SERVICE_TYPE - value: "flower" - - name: REDIS_URL - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/$(REDIS_DB)" - - name: CELERY_BROKER_URL - value: "redis://$(REDIS_HOST):$(REDIS_PORT)/0" - envFrom: - - configMapRef: - name: apple-exchange-config - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "256Mi" - cpu: "200m" - livenessProbe: - httpGet: - path: / - port: 5555 - initialDelaySeconds: 30 - periodSeconds: 30 - readinessProbe: - httpGet: - path: / - port: 5555 - initialDelaySeconds: 10 - periodSeconds: 10 - restartPolicy: Always diff --git a/backend/deploy/k8s/hpa.yaml b/backend/deploy/k8s/hpa.yaml deleted file mode 100644 index 62b48a6..0000000 --- a/backend/deploy/k8s/hpa.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Horizontal Pod Autoscaler for API服务 -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: apple-exchange-api-hpa - namespace: apple-exchange -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: apple-exchange-api - minReplicas: 3 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 - behavior: - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Percent - value: 10 - periodSeconds: 60 - scaleUp: - stabilizationWindowSeconds: 60 - policies: - - type: Percent - value: 50 - periodSeconds: 60 ---- -# Horizontal Pod Autoscaler for Worker服务 -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: apple-exchange-worker-hpa - namespace: apple-exchange -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: apple-exchange-worker - minReplicas: 2 - maxReplicas: 20 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 75 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 85 - behavior: - scaleDown: - stabilizationWindowSeconds: 600 # 更长的稳定窗口，避免频繁缩容 - policies: - - type: Percent - value: 25 - periodSeconds: 120 - scaleUp: - stabilizationWindowSeconds: 120 - policies: - - type: Percent - value: 100 - periodSeconds: 60 - - type: Pods - value: 5 - periodSeconds: 60 - selectPolicy: Max \ No newline at end of file diff --git a/backend/deploy/k8s/namespace.yaml b/backend/deploy/k8s/namespace.yaml deleted file mode 100644 index 47fc84a..0000000 --- a/backend/deploy/k8s/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: apple-exchange - labels: - name: apple-exchange - app: apple-exchange-system \ No newline at end of file diff --git a/backend/deploy/k8s/nodeport-access.md b/backend/deploy/k8s/nodeport-access.md deleted file mode 100644 index 505aa52..0000000 --- a/backend/deploy/k8s/nodeport-access.md +++ /dev/null @@ -1,34 +0,0 @@ -# NodePort 访问指南 - -## 服务访问 - -通过NodePort方式，可以直接通过Kubernetes集群节点的IP地址和指定端口访问服务。 - -### API服务 - -- 端口: 30080 -- 访问URL: http://<节点IP>:30080 - -### Flower监控服务 - -- 端口: 30555 -- 访问URL: http://<节点IP>:30555 - -## 使用说明 - -1. 获取集群节点IP: - ```bash - kubectl get nodes -o wide - ``` - -2. 使用任意节点的IP地址和对应的NodePort端口访问服务。 - -3. 如果在云环境中，请确保安全组/防火墙规则允许这些端口的入站流量。 - -## 安全注意事项 - -NodePort服务会在所有集群节点上开放指定端口，请确保: - -1. 在生产环境中配置适当的网络安全策略 -2. 考虑使用API网关或负载均衡器在NodePort前面提供额外的安全层 -3. 对敏感服务实施适当的认证和授权机制 diff --git a/backend/deploy/k8s/persistent-volumes.yaml b/backend/deploy/k8s/persistent-volumes.yaml deleted file mode 100644 index 9169a22..0000000 --- a/backend/deploy/k8s/persistent-volumes.yaml +++ /dev/null @@ -1,121 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: apple-exchange-screenshots-pv - labels: - app: apple-exchange - type: screenshots -spec: - capacity: - storage: 10Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - storageClassName: apple-exchange-storage - hostPath: - path: /data/apple-exchange/screenshots - type: DirectoryOrCreate ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: apple-exchange-logs-pv - labels: - app: apple-exchange - type: logs -spec: - capacity: - storage: 5Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - storageClassName: apple-exchange-storage - hostPath: - path: /data/apple-exchange/logs - type: DirectoryOrCreate ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: apple-exchange-shared-pv - labels: - app: apple-exchange - type: shared -spec: - capacity: - storage: 2Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - storageClassName: apple-exchange-storage - hostPath: - path: /data/apple-exchange/shared - type: DirectoryOrCreate ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: apple-exchange-screenshots-pvc - namespace: apple-exchange - labels: - app: apple-exchange - type: screenshots -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 10Gi - storageClassName: apple-exchange-storage - selector: - matchLabels: - app: apple-exchange - type: screenshots ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: apple-exchange-logs-pvc - namespace: apple-exchange - labels: - app: apple-exchange - type: logs -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 5Gi - storageClassName: apple-exchange-storage - selector: - matchLabels: - app: apple-exchange - type: logs ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: apple-exchange-shared-pvc - namespace: apple-exchange - labels: - app: apple-exchange - type: shared -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 2Gi - storageClassName: apple-exchange-storage - selector: - matchLabels: - app: apple-exchange - type: shared ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: apple-exchange-storage -provisioner: kubernetes.io/no-provisioner -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true diff --git a/backend/deploy/k8s/secret.yaml b/backend/deploy/k8s/secret.yaml deleted file mode 100644 index af01a6b..0000000 --- a/backend/deploy/k8s/secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: apple-exchange-secret - namespace: apple-exchange -type: Opaque -data: - # Base64编码的敏感信息 - # 使用: echo -n "your-password" | base64 - DATABASE_PASSWORD: cGFzc3dvcmQ= # password - REDIS_PASSWORD: "" # 空密码 - JWT_SECRET_KEY: eW91ci1qd3Qtc2VjcmV0LWtleQ== # your-jwt-secret-key - ENCRYPTION_KEY: eW91ci1lbmNyeXB0aW9uLWtleQ== # your-encryption-key \ No newline at end of file diff --git a/backend/deploy/k8s/service.yaml b/backend/deploy/k8s/service.yaml deleted file mode 100644 index 995aca7..0000000 --- a/backend/deploy/k8s/service.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# API服务 -apiVersion: v1 -kind: Service -metadata: - name: apple-exchange-api-service - namespace: apple-exchange - labels: - app: apple-exchange-api - component: api -spec: - type: ClusterIP - ports: - - port: 80 - targetPort: 8000 - protocol: TCP - name: http - selector: - app: apple-exchange-api - component: api ---- -# Flower监控服务 -apiVersion: v1 -kind: Service -metadata: - name: apple-exchange-flower-service - namespace: apple-exchange - labels: - app: apple-exchange-flower - component: flower -spec: - type: ClusterIP - ports: - - port: 5555 - targetPort: 5555 - protocol: TCP - name: flower - selector: - app: apple-exchange-flower - component: flower ---- -# PostgreSQL服务 -apiVersion: v1 -kind: Service -metadata: - name: postgres-service - namespace: apple-exchange - labels: - app: postgres -spec: - type: ClusterIP - ports: - - port: 5432 - targetPort: 5432 - protocol: TCP - name: postgres - selector: - app: postgres ---- -# Redis服务 -apiVersion: v1 -kind: Service -metadata: - name: redis-service - namespace: apple-exchange - labels: - app: redis -spec: - type: ClusterIP - ports: - - port: 6379 - targetPort: 6379 - protocol: TCP - name: redis - selector: - app: redis ---- -# API NodePort服务 (用于外部访问) -apiVersion: v1 -kind: Service -metadata: - name: apple-exchange-api-nodeport - namespace: apple-exchange - labels: - app: apple-exchange-api - component: api -spec: - type: NodePort - ports: - - port: 80 - targetPort: 8000 - nodePort: 30080 - protocol: TCP - name: http - selector: - app: apple-exchange-api - component: api ---- -# Flower监控NodePort服务 (用于外部访问) -apiVersion: v1 -kind: Service -metadata: - name: apple-exchange-flower-nodeport - namespace: apple-exchange - labels: - app: apple-exchange-flower - component: flower -spec: - type: NodePort - ports: - - port: 5555 - targetPort: 5555 - nodePort: 30555 - protocol: TCP - name: flower - selector: - app: apple-exchange-flower - component: flower diff --git a/backend/deploy/redis.conf b/backend/deploy/redis.conf deleted file mode 100644 index 20f82ce..0000000 --- a/backend/deploy/redis.conf +++ /dev/null @@ -1,61 +0,0 @@ -# Redis配置文件 - 针对分布式爬虫系统优化 - -# 基本配置 -bind 0.0.0.0 -port 6379 -timeout 0 -tcp-keepalive 300 - -# 内存配置 -maxmemory 512mb -maxmemory-policy allkeys-lru - -# 持久化配置 -save 900 1 -save 300 10 -save 60 10000 - -# AOF配置 -appendonly yes -appendfsync everysec -no-appendfsync-on-rewrite no -auto-aof-rewrite-percentage 100 -auto-aof-rewrite-min-size 64mb - -# 日志配置 -loglevel notice -logfile "" - -# 客户端配置 -maxclients 10000 - -# 慢查询日志 -slowlog-log-slower-than 10000 -slowlog-max-len 128 - -# 数据库数量 -databases 16 - -# 安全配置 -# requirepass your_password_here - -# 网络配置 -tcp-backlog 511 - -# 内存使用优化 -hash-max-ziplist-entries 512 -hash-max-ziplist-value 64 -list-max-ziplist-size -2 -list-compress-depth 0 -set-max-intset-entries 512 -zset-max-ziplist-entries 128 -zset-max-ziplist-value 64 - -# 分布式锁优化 -lua-time-limit 5000 - -# 键空间通知 (用于监控) -notify-keyspace-events Ex - -# 延迟监控 -latency-monitor-threshold 100 diff --git a/backend/docker-entrypoint-simple.sh b/backend/docker-entrypoint-simple.sh new file mode 100644 index 0000000..4f6bcf3 --- /dev/null +++ b/backend/docker-entrypoint-simple.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "Starting service..." +pwd +ls -la +exec "$@" \ No newline at end of file diff --git a/backend/deploy/docker-entrypoint.sh b/backend/docker-entrypoint.sh similarity index 73% rename from backend/deploy/docker-entrypoint.sh rename to backend/docker-entrypoint.sh index 3f238ed..c176b08 100644 --- a/backend/deploy/docker-entrypoint.sh +++ b/backend/docker-entrypoint.sh @@ -6,14 +6,6 @@ SERVICE_TYPE=${SERVICE_TYPE:-api} echo "Starting service type: $SERVICE_TYPE" -# 条件安装Playwright浏览器 - 仅在worker服务中安装 -if [ "$SERVICE_TYPE" = "worker" ]; then - echo "Installing Playwright browsers for worker service..." - /app/.venv/bin/playwright install chromium - /app/.venv/bin/playwright install-deps chromium - echo "Playwright browsers installed successfully" -fi - case "$SERVICE_TYPE" in "api") echo "Starting FastAPI server with Gunicorn..." @@ -39,13 +31,15 @@ case "$SERVICE_TYPE" in ;; "flower") echo "Starting Celery Flower monitoring..." - exec celery -A app.core.celery_app flower \ + exec /app/.venv/bin/celery -A app.core.celery_app flower \ --port=5555 \ - --broker=${REDIS_URL:-redis://redis:6379/0} + --broker=redis://redis:6379/0 \ + --result_backend=redis://redis:6379/1 \ + --loglevel=info ;; *) echo "Unknown service type: $SERVICE_TYPE" echo "Available types: api, worker, beat, flower" exit 1 ;; -esac +esac \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index eb6cc8c..67d7404 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -2,7 +2,7 @@ name = "apple-exchange-backend" version = "2.0.0" description = "Apple Gift Card Exchange Backend - FastAPI异步微服务架构" -readme = "README.md" +#readme = "README.md" requires-python = ">=3.13" authors = [ {name = "Apple Exchange Team", email = "team@apple-exchange.com"} @@ -19,105 +19,87 @@ classifiers = [ ] dependencies = [ - # OpenTelemetry 核心包 - "opentelemetry-api>=1.37.0", - "opentelemetry-sdk>=1.37.0", - - # OpenTelemetry 导出器 - "opentelemetry-exporter-otlp>=1.37.0", - "opentelemetry-exporter-otlp-proto-grpc>=1.37.0", - "opentelemetry-exporter-otlp-proto-http>=1.37.0", - - # OpenTelemetry Instrumentation - "opentelemetry-instrumentation>=0.58b0", - "opentelemetry-instrumentation-fastapi>=0.58b0", - "opentelemetry-instrumentation-sqlalchemy>=0.58b0", - "opentelemetry-instrumentation-redis>=0.58b0", - "opentelemetry-instrumentation-requests>=0.58b0", - "opentelemetry-instrumentation-httpx>=0.58b0", - "opentelemetry-instrumentation-logging>=0.58b0", - "opentelemetry-instrumentation-asyncpg>=0.58b0", - "opentelemetry-instrumentation-celery>=0.58b0", - "opentelemetry-instrumentation-system-metrics>=0.58b0", - - # OpenTelemetry 语义约定 - "opentelemetry-semantic-conventions>=0.58b0", - - # 核心框架和工具 - "python-multipart>=0.0.21", - "fastapi>=0.117.0", - "redis>=6.6.0", - "aioredis>=2.1.0", - "httpx>=0.29.0", - "aiohttp>=3.13.0", - "pydantic>=2.12.0", - "pydantic-settings>=2.11.0", - "alembic>=1.17.0", - "asyncpg>=0.31.0", - "aiosqlite>=0.22.0", - "uvicorn[standard]>=0.36.0", - "gunicorn>=24.0.0", - "sqlalchemy>=2.1.0", - "asyncio-mqtt>=0.17.0", - "playwright>=1.56.0", - "pandas>=2.4.0", - "openpyxl>=3.2.0", - "xlsxwriter>=3.3.0", - "structlog>=26.0.0", - "python-json-logger>=3.4.0", - "python-dotenv>=1.2.0", - "python-jose[cryptography]>=3.6.0", - "passlib[bcrypt]>=1.8.0", - "celery>=5.6.0", - "click>=8.3.0", - "rich>=15.0.0", - "typer>=0.17.0", - "psutil>=7.1.0", - "black>=26.0.0", - "isort>=6.1.0", - "flake8>=8.0.0", - "mypy>=1.18.0", - "pytest>=9.0.0", - "pytest-asyncio>=1.2.0", - "pytest-cov>=6.3.0", - "pre-commit>=5.0.0", - "aiofiles>=25.0.0", - "pandas-stubs>=2.4.0.241231", - "kombu>=5.6.0", - "gevent>=26.0.0", - "loguru>=0.8.0", - - # gRPC 核心工具 - "grpcio>=1.70.0", - "grpcio-status>=1.70.0", - "protobuf>=5.30.0", - - # 性能监控 - "prometheus-client>=0.22.0", + "opentelemetry-instrumentation-fastapi>=0.57b0", + "opentelemetry-api>=1.36.0", + "python-multipart>=0.0.20", + "opentelemetry-sdk>=1.36.0", + "opentelemetry-exporter-otlp>=1.36.0", + "opentelemetry-instrumentation-sqlalchemy>=0.57b0", + "opentelemetry-instrumentation-redis>=0.57b0", + "opentelemetry-instrumentation-requests>=0.57b0", + "opentelemetry-instrumentation-httpx>=0.57b0", + "fastapi>=0.116.1", + "redis>=6.4.0", + "aioredis>=2.0.1", + "httpx>=0.28.1", + "aiohttp>=3.12.15", + "pydantic>=2.11.7", + "pydantic-settings>=2.10.1", + "alembic>=1.16.4", + "asyncpg>=0.30.0", + "aiosqlite>=0.21.0", + "psycopg2-binary>=2.9.10", + "uvicorn[standard]>=0.35.0", + "gunicorn>=23.0.0", + "sqlalchemy>=2.0.42", + "asyncio-mqtt>=0.16.2", + "playwright>=1.54.0", + "pandas>=2.3.2", + "openpyxl>=3.1.5", + "xlsxwriter>=3.2.5", + "structlog>=25.4.0", + "python-json-logger>=3.3.0", + "python-dotenv>=1.1.1", + "python-jose[cryptography]>=3.5.0", + "passlib[bcrypt]>=1.7.4", + "click>=8.2.1", + "rich>=14.1.0", + "typer>=0.16.1", + "psutil>=7.0.0", + "black>=25.1.0", + "isort>=6.0.1", + "flake8>=7.3.0", + "mypy>=1.17.1", + "pytest>=8.4.1", + "pytest-asyncio>=1.1.0", + "pytest-cov>=6.2.1", + "pre-commit>=4.3.0", + "opentelemetry-instrumentation-logging>=0.57b0", + "aiofiles>=24.1.0", + "pandas-stubs>=2.3.0.250703", + "kombu>=5.5.4", + "gevent>=25.5.1", + "loguru>=0.7.3", + "opentelemetry-instrumentation-asyncpg>=0.57b0", + "opentelemetry-instrumentation-celery>=0.57b0", + "opentelemetry-instrumentation-system-metrics>=0.57b0", + "celery>=5.5.3", + "flower>=2.0.1", + "build>=1.3.0", ] [project.optional-dependencies] dev = [ - "pytest>=9.0.0", - "pytest-asyncio>=1.2.0", - "pytest-cov>=6.3.0", - "pytest-mock>=3.13.0", - "httpx>=0.29.0", - "faker>=21.0.0", + "pytest>=7.4.3", + "pytest-asyncio>=0.21.1", + "pytest-cov>=4.1.0", + "pytest-mock>=3.12.0", + "httpx>=0.25.2", + "faker>=20.1.0", ] test = [ - "pytest>=9.0.0", - "pytest-asyncio>=1.2.0", - "pytest-cov>=6.3.0", - "pytest-mock>=3.13.0", - "coverage>=7.4.0", + "pytest>=7.4.3", + "pytest-asyncio>=0.21.1", + "pytest-cov>=4.1.0", + "pytest-mock>=3.12.0", + "coverage>=7.3.2", ] docs = [ - "mkdocs>=1.6.0", - "mkdocs-material>=9.7.0", - "mkdocstrings[python]>=0.26.0", + "mkdocs>=1.5.3", + "mkdocs-material>=9.4.8", + "mkdocstrings[python]>=0.24.0", ] [project.urls] diff --git a/backend/run.py b/backend/run.py new file mode 100644 index 0000000..d9fe00e --- /dev/null +++ b/backend/run.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +""" +Gunicorn配置文件 +用于生产环境的FastAPI应用启动 +""" + +import os +import multiprocessing +from pathlib import Path + +# 基础配置 +bind = "0.0.0.0:8000" +workers = int(os.environ.get("WORKERS", multiprocessing.cpu_count() * 2 + 1)) +worker_class = "uvicorn.workers.UvicornWorker" +worker_connections = 1000 +max_requests = 1000 +max_requests_jitter = 100 +timeout = 30 +keepalive = 2 + +# 日志配置 +accesslog = "-" +errorlog = "-" +loglevel = "info" + +# 安全配置 +limit_request_line = 4096 +limit_request_fields = 100 +limit_request_field_size = 8190 + +# 进程管理 +preload_app = True +pidfile = "/tmp/gunicorn.pid" + +# 工作进程重启时的临时目录 +temp_dir = "/tmp" + +# 工作进程重启时保留的文件描述符数量 +worker_tmp_dir = "/dev/shm" + +# 环境变量传递 +raw_env = [ + f"ENVIRONMENT={os.environ.get('ENVIRONMENT', 'production')}", + f"DATABASE_URL={os.environ.get('DATABASE_URL', '')}", + f"REDIS_URL={os.environ.get('REDIS_URL', '')}", + f"CELERY_BROKER_URL={os.environ.get('CELERY_BROKER_URL', '')}", + f"CELERY_RESULT_BACKEND={os.environ.get('CELERY_RESULT_BACKEND', '')}", +] + +# 重启策略 +graceful_timeout = 30 +reload = False + +# 统计和监控 +statsd_host = None +statsd_prefix = "" + +# 工作进程启动时的钩子函数 +def on_starting(server): + """工作进程启动前的钩子函数""" + pass + +def on_reload(server): + """重载时的钩子函数""" + pass + +def when_ready(server): + """服务器准备就绪时的钩子函数""" + pass + +def on_exit(server): + """服务器退出时的钩子函数""" + pass + +# 工作进程相关的钩子函数 +def pre_fork(server, worker): + """工作进程fork前的钩子函数""" + pass + +def post_fork(server, worker): + """工作进程fork后的钩子函数""" + pass + +def post_worker_init(worker): + """工作进程初始化后的钩子函数""" + pass + +def worker_int(worker): + """工作进程收到中断信号时的钩子函数""" + pass + +def worker_abort(worker): + """工作进程异常终止时的钩子函数""" + pass + +def child_exit(server, worker): + """子进程退出时的钩子函数""" + pass + +def pre_exec(server): + """工作进程执行前的钩子函数""" + pass + +def pre_request(worker, req): + """请求处理前的钩子函数""" + pass + +def post_request(worker, req, environ, resp): + """请求处理后的钩子函数""" + pass \ No newline at end of file diff --git a/backend/test_gunicorn.py b/backend/test_gunicorn.py new file mode 100644 index 0000000..bc04297 --- /dev/null +++ b/backend/test_gunicorn.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +""" +简单的Gunicorn测试脚本 +用于验证Docker容器中的Gunicorn配置 +""" + +import os +import sys +import signal +import time +import multiprocessing +from pathlib import Path + +# 模拟Gunicorn配置 +bind = "0.0.0.0:8000" +workers = max(1, multiprocessing.cpu_count() * 2 + 1) +worker_class = "uvicorn.workers.UvicornWorker" +worker_connections = 1000 +timeout = 30 +keepalive = 2 + +def test_imports(): + """测试必要的模块导入""" + try: + import uvicorn + import gunicorn + print("✅ Uvicorn和Gunicorn导入成功") + return True + except ImportError as e: + print(f"❌ 模块导入失败: {e}") + return False + +def test_celery(): + """测试Celery应用""" + try: + from app.core.celery_app import get_celery_app + app = get_celery_app() + print("✅ Celery应用创建成功") + return True + except ImportError as e: + print(f"❌ Celery应用创建失败: {e}") + return False + +def test_database(): + """测试数据库连接""" + try: + from app.core.database import get_database + db = get_database() + print("✅ 数据库连接配置成功") + return True + except ImportError as e: + print(f"❌ 数据库配置失败: {e}") + return False + +def main(): + """主测试函数""" + print("🚀 开始测试Docker容器环境...") + + tests = [ + test_imports, + test_celery, + test_database, + ] + + passed = 0 + failed = 0 + + for test in tests: + try: + if test(): + passed += 1 + else: + failed += 1 + except Exception as e: + print(f"❌ 测试异常: {e}") + failed += 1 + + print(f"\n📊 测试结果: {passed} 通过, {failed} 失败") + + if failed == 0: + print("🎉 所有测试通过! 容器准备就绪。") + return 0 + else: + print("⚠️ 有测试失败，请检查配置。") + return 1 + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/backend/test_tasks.py b/backend/test_tasks.py new file mode 100644 index 0000000..edaf6ad --- /dev/null +++ b/backend/test_tasks.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +""" +测试 Celery 任务注册 +""" + +import sys +from pathlib import Path + +# 添加项目根目录到Python路径 +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +def test_task_registration(): + """测试任务注册""" + try: + print("正在导入 Celery 应用...") + from app.core.celery_app import celery_app + + print("正在导入任务模块...") + from app.tasks import crawler_tasks + + print("检查已注册的任务...") + registered_tasks = list(celery_app.tasks.keys()) + + print(f"已注册的任务数量: {len(registered_tasks)}") + print("已注册的任务:") + for task_name in sorted(registered_tasks): + if not task_name.startswith('celery.'): + print(f" ✓ {task_name}") + + # 检查特定任务 + target_tasks = [ + 'app.tasks.crawler_tasks.batch_process_orders', + 'app.tasks.crawler_tasks.process_apple_order' + ] + + print("\n检查目标任务:") + for task_name in target_tasks: + if task_name in celery_app.tasks: + print(f" ✓ {task_name} - 已注册") + else: + print(f" ✗ {task_name} - 未注册") + return False + + print("\n✅ 所有任务都已正确注册!") + return True + + except Exception as e: + print(f"❌ 任务注册测试失败: {e}") + import traceback + traceback.print_exc() + return False + +if __name__ == "__main__": + success = test_task_registration() + sys.exit(0 if success else 1) diff --git a/backend/uv.lock b/backend/uv.lock index 591feb1..c8d5373 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -152,13 +152,13 @@ dependencies = [ { name = "asyncio-mqtt" }, { name = "asyncpg" }, { name = "black" }, + { name = "build" }, { name = "celery" }, { name = "click" }, { name = "fastapi" }, { name = "flake8" }, + { name = "flower" }, { name = "gevent" }, - { name = "grpcio" }, - { name = "grpcio-status" }, { name = "gunicorn" }, { name = "httpx" }, { name = "isort" }, @@ -168,9 +168,6 @@ dependencies = [ { name = "openpyxl" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp" }, - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-instrumentation" }, { name = "opentelemetry-instrumentation-asyncpg" }, { name = "opentelemetry-instrumentation-celery" }, { name = "opentelemetry-instrumentation-fastapi" }, @@ -181,15 +178,13 @@ dependencies = [ { name = "opentelemetry-instrumentation-sqlalchemy" }, { name = "opentelemetry-instrumentation-system-metrics" }, { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions" }, { name = "pandas" }, { name = "pandas-stubs" }, { name = "passlib", extra = ["bcrypt"] }, { name = "playwright" }, { name = "pre-commit" }, - { name = "prometheus-client" }, - { name = "protobuf" }, { name = "psutil" }, + { name = "psycopg2-binary" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pytest" }, @@ -240,15 +235,15 @@ requires-dist = [ { name = "asyncio-mqtt", specifier = ">=0.16.2" }, { name = "asyncpg", specifier = ">=0.30.0" }, { name = "black", specifier = ">=25.1.0" }, + { name = "build", specifier = ">=1.3.0" }, { name = "celery", specifier = ">=5.5.3" }, { name = "click", specifier = ">=8.2.1" }, { name = "coverage", marker = "extra == 'test'", specifier = ">=7.3.2" }, { name = "faker", marker = "extra == 'dev'", specifier = ">=20.1.0" }, { name = "fastapi", specifier = ">=0.116.1" }, { name = "flake8", specifier = ">=7.3.0" }, + { name = "flower", specifier = ">=2.0.1" }, { name = "gevent", specifier = ">=25.5.1" }, - { name = "grpcio", specifier = ">=1.68.1" }, - { name = "grpcio-status", specifier = ">=1.68.1" }, { name = "gunicorn", specifier = ">=23.0.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.25.2" }, @@ -262,9 +257,6 @@ requires-dist = [ { name = "openpyxl", specifier = ">=3.1.5" }, { name = "opentelemetry-api", specifier = ">=1.36.0" }, { name = "opentelemetry-exporter-otlp", specifier = ">=1.36.0" }, - { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.36.0" }, - { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.36.0" }, - { name = "opentelemetry-instrumentation", specifier = ">=0.57b0" }, { name = "opentelemetry-instrumentation-asyncpg", specifier = ">=0.57b0" }, { name = "opentelemetry-instrumentation-celery", specifier = ">=0.57b0" }, { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.57b0" }, @@ -275,15 +267,13 @@ requires-dist = [ { name = "opentelemetry-instrumentation-sqlalchemy", specifier = ">=0.57b0" }, { name = "opentelemetry-instrumentation-system-metrics", specifier = ">=0.57b0" }, { name = "opentelemetry-sdk", specifier = ">=1.36.0" }, - { name = "opentelemetry-semantic-conventions", specifier = ">=0.57b0" }, { name = "pandas", specifier = ">=2.3.2" }, { name = "pandas-stubs", specifier = ">=2.3.0.250703" }, { name = "passlib", extras = ["bcrypt"], specifier = ">=1.7.4" }, { name = "playwright", specifier = ">=1.54.0" }, { name = "pre-commit", specifier = ">=4.3.0" }, - { name = "prometheus-client", specifier = ">=0.21.0" }, - { name = "protobuf", specifier = ">=5.28.3" }, { name = "psutil", specifier = ">=7.0.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.10" }, { name = "pydantic", specifier = ">=2.11.7" }, { name = "pydantic-settings", specifier = ">=2.10.1" }, { name = "pytest", specifier = ">=8.4.1" }, @@ -467,6 +457,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646, upload-time = "2025-01-29T04:15:38.082Z" }, ] +[[package]] +name = "build" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "os_name == 'nt'" }, + { name = "packaging" }, + { name = "pyproject-hooks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/1c/23e33405a7c9eac261dff640926b8b5adaed6a6eb3e1767d441ed611d0c0/build-1.3.0.tar.gz", hash = "sha256:698edd0ea270bde950f53aed21f3a0135672206f3911e0176261a31e0e07b397", size = 48544, upload-time = "2025-08-01T21:27:09.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/8c/2b30c12155ad8de0cf641d76a8b396a16d2c36bc6d50b621a62b7c4567c1/build-1.3.0-py3-none-any.whl", hash = "sha256:7145f0b5061ba90a1500d60bd1b13ca0a8a4cebdd0cc16ed8adf1c0e739f43b4", size = 23382, upload-time = "2025-08-01T21:27:07.844Z" }, +] + [[package]] name = "celery" version = "5.5.3" @@ -773,6 +777,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/56/13ab06b4f93ca7cac71078fbe37fcea175d3216f31f85c3168a6bbd0bb9a/flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e", size = 57922, upload-time = "2025-06-20T19:31:34.425Z" }, ] +[[package]] +name = "flower" +version = "2.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "celery" }, + { name = "humanize" }, + { name = "prometheus-client" }, + { name = "pytz" }, + { name = "tornado" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/a1/357f1b5d8946deafdcfdd604f51baae9de10aafa2908d0b7322597155f92/flower-2.0.1.tar.gz", hash = "sha256:5ab717b979530770c16afb48b50d2a98d23c3e9fe39851dcf6bc4d01845a02a0", size = 3220408, upload-time = "2023-08-13T14:37:46.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/ff/ee2f67c0ff146ec98b5df1df637b2bc2d17beeb05df9f427a67bd7a7d79c/flower-2.0.1-py2.py3-none-any.whl", hash = "sha256:9db2c621eeefbc844c8dd88be64aef61e84e2deb29b271e02ab2b5b9f01068e2", size = 383553, upload-time = "2023-08-13T14:37:41.552Z" }, +] + [[package]] name = "frozenlist" version = "1.7.0" @@ -917,20 +937,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/34/80/de3eb55eb581815342d097214bed4c59e806b05f1b3110df03b2280d6dfd/grpcio-1.74.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24", size = 4489214, upload-time = "2025-07-24T18:53:59.771Z" }, ] -[[package]] -name = "grpcio-status" -version = "1.74.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/22/238c5f01e6837df54494deb08d5c772bc3f5bf5fb80a15dce254892d1a81/grpcio_status-1.74.0.tar.gz", hash = "sha256:c58c1b24aa454e30f1fc6a7e0dbbc194c54a408143971a94b5f4e40bb5831432", size = 13662, upload-time = "2025-07-24T19:01:56.874Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/aa/1b1fe7d8ab699e1ec26d3a36b91d3df9f83a30abc07d4c881d0296b17b67/grpcio_status-1.74.0-py3-none-any.whl", hash = "sha256:52cdbd759a6760fc8f668098a03f208f493dd5c76bf8e02598bbbaf1f6fc2876", size = 14425, upload-time = "2025-07-24T19:01:19.963Z" }, -] - [[package]] name = "gunicorn" version = "23.0.0" @@ -995,6 +1001,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "humanize" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/1d/3062fcc89ee05a715c0b9bfe6490c00c576314f27ffee3a704122c6fd259/humanize-4.13.0.tar.gz", hash = "sha256:78f79e68f76f0b04d711c4e55d32bebef5be387148862cb1ef83d2b58e7935a0", size = 81884, upload-time = "2025-08-25T09:39:20.04Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/c7/316e7ca04d26695ef0635dc81683d628350810eb8e9b2299fc08ba49f366/humanize-4.13.0-py3-none-any.whl", hash = "sha256:b810820b31891813b1673e8fec7f1ed3312061eab2f26e3fa192c393d11ed25f", size = 128869, upload-time = "2025-08-25T09:39:18.54Z" }, +] + [[package]] name = "identify" version = "2.6.13" @@ -1884,14 +1899,14 @@ wheels = [ [[package]] name = "prompt-toolkit" -version = "3.0.51" +version = "3.0.52" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wcwidth" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/6e/9d084c929dfe9e3bfe0c6a47e31f78a25c54627d64a66e884a8bf5474f1c/prompt_toolkit-3.0.51.tar.gz", hash = "sha256:931a162e3b27fc90c86f1b48bb1fb2c528c2761475e57c9c06de13311c7b54ed", size = 428940, upload-time = "2025-04-15T09:18:47.731Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/4f/5249960887b1fbe561d9ff265496d170b55a735b76724f10ef19f9e40716/prompt_toolkit-3.0.51-py3-none-any.whl", hash = "sha256:52742911fde84e2d423e2f9a4cf1de7d7ac4e51958f648d9540e0fb8db077b07", size = 387810, upload-time = "2025-04-15T09:18:44.753Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, ] [[package]] @@ -1964,6 +1979,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885, upload-time = "2025-02-13T21:54:37.486Z" }, ] +[[package]] +name = "psycopg2-binary" +version = "2.9.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/bdc8274dc0585090b4e3432267d7be4dfbfd8971c0fa59167c711105a6bf/psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2", size = 385764, upload-time = "2024-10-16T11:24:58.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/d41d3ba765609c0763505d565c4d12d8f3c79793f0d0f044ff5a28bf395b/psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d", size = 3044699, upload-time = "2024-10-16T11:21:42.841Z" }, + { url = "https://files.pythonhosted.org/packages/35/44/257ddadec7ef04536ba71af6bc6a75ec05c5343004a7ec93006bee66c0bc/psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb", size = 3275245, upload-time = "2024-10-16T11:21:51.989Z" }, + { url = "https://files.pythonhosted.org/packages/1b/11/48ea1cd11de67f9efd7262085588790a95d9dfcd9b8a687d46caf7305c1a/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7", size = 2851631, upload-time = "2024-10-16T11:21:57.584Z" }, + { url = "https://files.pythonhosted.org/packages/62/e0/62ce5ee650e6c86719d621a761fe4bc846ab9eff8c1f12b1ed5741bf1c9b/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d", size = 3082140, upload-time = "2024-10-16T11:22:02.005Z" }, + { url = "https://files.pythonhosted.org/packages/27/ce/63f946c098611f7be234c0dd7cb1ad68b0b5744d34f68062bb3c5aa510c8/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73", size = 3264762, upload-time = "2024-10-16T11:22:06.412Z" }, + { url = "https://files.pythonhosted.org/packages/43/25/c603cd81402e69edf7daa59b1602bd41eb9859e2824b8c0855d748366ac9/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673", size = 3020967, upload-time = "2024-10-16T11:22:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/5f/d6/8708d8c6fca531057fa170cdde8df870e8b6a9b136e82b361c65e42b841e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f", size = 2872326, upload-time = "2024-10-16T11:22:16.406Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ac/5b1ea50fc08a9df82de7e1771537557f07c2632231bbab652c7e22597908/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909", size = 2822712, upload-time = "2024-10-16T11:22:21.366Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fc/504d4503b2abc4570fac3ca56eb8fed5e437bf9c9ef13f36b6621db8ef00/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1", size = 2920155, upload-time = "2024-10-16T11:22:25.684Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d1/323581e9273ad2c0dbd1902f3fb50c441da86e894b6e25a73c3fda32c57e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567", size = 2959356, upload-time = "2024-10-16T11:22:30.562Z" }, + { url = "https://files.pythonhosted.org/packages/08/50/d13ea0a054189ae1bc21af1d85b6f8bb9bbc5572991055d70ad9006fe2d6/psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142", size = 2569224, upload-time = "2025-01-04T20:09:19.234Z" }, +] + [[package]] name = "pyasn1" version = "0.6.1" @@ -2091,6 +2125,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/06/43084e6cbd4b3bc0e80f6be743b2e79fbc6eed8de9ad8c629939fa55d972/pymdown_extensions-10.16.1-py3-none-any.whl", hash = "sha256:d6ba157a6c03146a7fb122b2b9a121300056384eafeec9c9f9e584adfdb2a32d", size = 266178, upload-time = "2025-07-28T16:19:31.401Z" }, ] +[[package]] +name = "pyproject-hooks" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/82/28175b2414effca1cdac8dc99f76d660e7a4fb0ceefa4b4ab8f5f6742925/pyproject_hooks-1.2.0.tar.gz", hash = "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8", size = 19228, upload-time = "2024-09-29T09:24:13.293Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, +] + [[package]] name = "pytest" version = "8.4.1" @@ -2368,6 +2411,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/4a/97ee6973e3a73c74c8120d59829c3861ea52210667ec3e7a16045c62b64d/structlog-25.4.0-py3-none-any.whl", hash = "sha256:fe809ff5c27e557d14e613f45ca441aabda051d119ee5a0102aaba6ce40eed2c", size = 68720, upload-time = "2025-06-02T08:21:11.43Z" }, ] +[[package]] +name = "tornado" +version = "6.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/ce/1eb500eae19f4648281bb2186927bb062d2438c2e5093d1360391afd2f90/tornado-6.5.2.tar.gz", hash = "sha256:ab53c8f9a0fa351e2c0741284e06c7a45da86afb544133201c5cc8578eb076a0", size = 510821, upload-time = "2025-08-08T18:27:00.78Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/48/6a7529df2c9cc12efd2e8f5dd219516184d703b34c06786809670df5b3bd/tornado-6.5.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:2436822940d37cde62771cff8774f4f00b3c8024fe482e16ca8387b8a2724db6", size = 442563, upload-time = "2025-08-08T18:26:42.945Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b5/9b575a0ed3e50b00c40b08cbce82eb618229091d09f6d14bce80fc01cb0b/tornado-6.5.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:583a52c7aa94ee046854ba81d9ebb6c81ec0fd30386d96f7640c96dad45a03ef", size = 440729, upload-time = "2025-08-08T18:26:44.473Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4e/619174f52b120efcf23633c817fd3fed867c30bff785e2cd5a53a70e483c/tornado-6.5.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0fe179f28d597deab2842b86ed4060deec7388f1fd9c1b4a41adf8af058907e", size = 444295, upload-time = "2025-08-08T18:26:46.021Z" }, + { url = "https://files.pythonhosted.org/packages/95/fa/87b41709552bbd393c85dd18e4e3499dcd8983f66e7972926db8d96aa065/tornado-6.5.2-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b186e85d1e3536d69583d2298423744740986018e393d0321df7340e71898882", size = 443644, upload-time = "2025-08-08T18:26:47.625Z" }, + { url = "https://files.pythonhosted.org/packages/f9/41/fb15f06e33d7430ca89420283a8762a4e6b8025b800ea51796ab5e6d9559/tornado-6.5.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e792706668c87709709c18b353da1f7662317b563ff69f00bab83595940c7108", size = 443878, upload-time = "2025-08-08T18:26:50.599Z" }, + { url = "https://files.pythonhosted.org/packages/11/92/fe6d57da897776ad2e01e279170ea8ae726755b045fe5ac73b75357a5a3f/tornado-6.5.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:06ceb1300fd70cb20e43b1ad8aaee0266e69e7ced38fa910ad2e03285009ce7c", size = 444549, upload-time = "2025-08-08T18:26:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/9b/02/c8f4f6c9204526daf3d760f4aa555a7a33ad0e60843eac025ccfd6ff4a93/tornado-6.5.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:74db443e0f5251be86cbf37929f84d8c20c27a355dd452a5cfa2aada0d001ec4", size = 443973, upload-time = "2025-08-08T18:26:53.625Z" }, + { url = "https://files.pythonhosted.org/packages/ae/2d/f5f5707b655ce2317190183868cd0f6822a1121b4baeae509ceb9590d0bd/tornado-6.5.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b5e735ab2889d7ed33b32a459cac490eda71a1ba6857b0118de476ab6c366c04", size = 443954, upload-time = "2025-08-08T18:26:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/e8/59/593bd0f40f7355806bf6573b47b8c22f8e1374c9b6fd03114bd6b7a3dcfd/tornado-6.5.2-cp39-abi3-win32.whl", hash = "sha256:c6f29e94d9b37a95013bb669616352ddb82e3bfe8326fccee50583caebc8a5f0", size = 445023, upload-time = "2025-08-08T18:26:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/f609b420c2f564a748a2d80ebfb2ee02a73ca80223af712fca591386cafb/tornado-6.5.2-cp39-abi3-win_amd64.whl", hash = "sha256:e56a5af51cc30dd2cae649429af65ca2f6571da29504a07995175df14c18f35f", size = 445427, upload-time = "2025-08-08T18:26:57.91Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4f/e1f65e8f8c76d73658b33d33b81eed4322fb5085350e4328d5c956f0c8f9/tornado-6.5.2-cp39-abi3-win_arm64.whl", hash = "sha256:d6c33dc3672e3a1f3618eb63b7ef4683a7688e7b9e6e8f0d9aa5726360a004af", size = 444456, upload-time = "2025-08-08T18:26:59.207Z" }, +] + [[package]] name = "typer" version = "0.16.1" diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..060740c --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,272 @@ +# Apple Gift Card Exchange Platform - 部署指南 + +本目录包含完整的前后端部署配置文件。 + +## 🚀 快速开始 + +### 1. 环境要求 +- Docker >= 20.10 +- Docker Compose >= 2.0 +- 至少 4GB 内存 +- 至少 10GB 磁盘空间 + +### 2. 一键部署 + +```bash +# 开发环境部署 +./deploy/deploy.sh dev + +# 生产环境部署 +./deploy/deploy.sh prod + +# 包含监控的生产环境部署 +./deploy/deploy.sh monitoring + +# 清理并重新部署 +./deploy/deploy.sh dev --cleanup +``` + +### 3. 手动部署 + +```bash +# 创建必要目录 +mkdir -p deploy/ssl deploy/logs + +# 启动开发环境 +docker-compose -f deploy/docker-compose.combined.yml up -d + +# 启动生产环境（包含 Nginx） +docker-compose -f deploy/docker-compose.combined.yml --profile production up -d + +# 启动监控服务 +docker-compose -f deploy/docker-compose.combined.yml --profile monitoring up -d +``` + +## 📁 文件结构 + +``` +deploy/ +├── docker-compose.yml # 后端服务配置 +├── docker-compose.combined.yml # 完整前后端配置 +├── Dockerfile # 后端 API/Beat/Flower 镜像 +├── Dockerfile.worker # 后端 Worker 镜像 +├── Dockerfile.frontend # 前端 Next.js 镜像 +├── nginx.conf # Nginx 配置 +├── redis.conf # Redis 配置 +├── docker-entrypoint.sh # 后端启动脚本 +├── run.py # 后端运行脚本 +├── test_gunicorn.py # Gunicorn 测试脚本 +├── deploy.sh # 自动化部署脚本 +├── k8s/ # Kubernetes 配置 +├── helm/ # Helm Charts +└── ssl/ # SSL 证书目录 +``` + +## 🔧 服务配置 + +### 前端服务 (Next.js) +- **端口**: 3000 +- **镜像**: 自定义构建 +- **环境变量**: + - `NEXT_PUBLIC_API_URL`: 后端 API 地址 + - `NEXT_PUBLIC_ENV`: 环境类型 + - `NEXT_PUBLIC_TELEMETRY_ENDPOINT`: 遥测端点 + +### 后端服务 (FastAPI) +- **端口**: 8000 +- **镜像**: 自定义构建 +- **服务类型**: api, worker, beat, flower +- **数据库**: PostgreSQL 17 +- **缓存**: Redis 7 + +### 基础设施 +- **数据库**: PostgreSQL 17 +- **缓存**: Redis 7 +- **消息队列**: Redis (Celery) +- **代理**: Nginx (生产环境) + +## 🌐 访问地址 + +### 开发环境 +- 前端: http://localhost:3000 +- 后端 API: http://localhost:8000 +- API 文档: http://localhost:8000/docs + +### 生产环境 +- 前端: http://localhost +- 后端 API: http://localhost/api +- API 文档: http://localhost/api/docs + +### 监控服务 +- Celery Flower: http://localhost:5555 + +## 📊 健康检查 + +```bash +# 前端健康检查 +curl http://localhost:3000/ + +# 后端健康检查 +curl http://localhost:8000/api/v1/health/liveness + +# 数据库检查 +docker exec apple-exchange-db pg_isready -U postgres + +# Redis 检查 +docker exec apple-exchange-redis redis-cli ping +``` + +## 🔍 日志管理 + +```bash +# 查看所有服务日志 +docker-compose -f deploy/docker-compose.combined.yml logs -f + +# 查看特定服务日志 +docker-compose -f deploy/docker-compose.combined.yml logs -f frontend +docker-compose -f deploy/docker-compose.combined.yml logs -f api +docker-compose -f deploy/docker-compose.combined.yml logs -f worker + +# 查看错误日志 +docker-compose -f deploy/docker-compose.combined.yml logs --tail=100 +``` + +## 🛠️ 常用操作 + +```bash +# 停止所有服务 +docker-compose -f deploy/docker-compose.combined.yml down + +# 重启特定服务 +docker-compose -f deploy/docker-compose.combined.yml restart api + +# 扩展 Worker 服务 +docker-compose -f deploy/docker-compose.combined.yml up -d --scale worker=3 + +# 清理未使用的资源 +docker system prune -f + +# 查看资源使用情况 +docker stats +``` + +## 🔒 安全配置 + +### SSL 证书 +生产环境需要配置 SSL 证书： + +```bash +# 生成自签名证书（仅用于测试） +openssl req -x509 -newkey rsa:4096 -keyout deploy/ssl/key.pem -out deploy/ssl/cert.pem -days 365 -nodes + +# 使用 Let's Encrypt（生产环境） +# 需要配置域名和反向代理 +``` + +### 环境变量 +创建环境文件： + +```bash +# 后端环境变量 +cp backend/.env.example backend/.env + +# 前端环境变量 +cp frontend/env.example frontend/.env.local +``` + +## 📈 性能优化 + +### 数据库优化 +```sql +-- 在 PostgreSQL 中执行 +CREATE INDEX idx_orders_status ON orders(status); +CREATE INDEX idx_orders_created_at ON orders(created_at); +ANALYZE; +``` + +### Redis 优化 +```bash +# 在 redis.conf 中配置 +maxmemory 512mb +maxmemory-policy allkeys-lru +save 900 1 +save 300 10 +save 60 10000 +``` + +### Docker 优化 +```yaml +# 在 docker-compose.yml 中调整资源限制 +deploy: + resources: + limits: + cpus: '2.0' + memory: 2G + reservations: + cpus: '1.0' + memory: 1G +``` + +## 🚨 故障排除 + +### 常见问题 + +1. **端口冲突** + ```bash + # 检查端口占用 + netstat -tulpn | grep :3000 + netstat -tulpn | grep :8000 + ``` + +2. **内存不足** + ```bash + # 清理 Docker 资源 + docker system prune -a -f + ``` + +3. **数据库连接失败** + ```bash + # 重启数据库 + docker-compose -f deploy/docker-compose.combined.yml restart db + ``` + +4. **Worker 任务堆积** + ```bash + # 扩展 Worker 数量 + docker-compose -f deploy/docker-compose.combined.yml up -d --scale worker=5 + ``` + +### 日志分析 +```bash +# 查看错误日志 +docker-compose -f deploy/docker-compose.combined.yml logs --tail=100 api | grep ERROR + +# 查看慢查询 +docker-compose -f deploy/docker-compose.combined.yml logs --tail=100 db | grep SLOW +``` + +## 🔄 更新部署 + +```bash +# 拉取最新代码 +git pull origin main + +# 重新构建并启动 +docker-compose -f deploy/docker-compose.combined.yml build +docker-compose -f deploy/docker-compose.combined.yml up -d + +# 数据库迁移（如果需要） +docker-compose -f deploy/docker-compose.combined.yml exec api alembic upgrade head +``` + +## 📞 支持 + +如果遇到问题，请： +1. 查看日志文件 +2. 检查服务状态 +3. 查看本项目文档 +4. 提交 Issue + +--- + +**注意**: 生产环境部署前请仔细阅读安全配置和性能优化部分。 \ No newline at end of file diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100644 index 0000000..89a56f1 --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,204 @@ +#!/bin/bash + +# Apple Gift Card Exchange Platform - 部署脚本 +# 支持开发环境和生产环境的 Docker 部署 + +set -e # 遇到错误立即退出 + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# 打印带颜色的信息 +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# 检查 Docker 是否安装 +check_docker() { + if ! command -v docker &> /dev/null; then + print_error "Docker 未安装，请先安装 Docker" + exit 1 + fi + if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then + print_error "Docker Compose 未安装，请先安装 Docker Compose" + exit 1 + fi +} + +# 检查环境文件 +check_env_files() { + if [ ! -f "backend/.env" ]; then + print_warning "backend/.env 文件不存在，使用默认配置" + cp backend/.env.example backend/.env 2>/dev/null || true + fi + + if [ ! -f "frontend/.env.local" ]; then + print_warning "frontend/.env.local 文件不存在，使用默认配置" + cp frontend/env.example frontend/.env.local 2>/dev/null || true + fi +} + +# 创建必要的目录 +create_directories() { + print_info "创建必要的目录..." + mkdir -p deploy/ssl + mkdir -p deploy/logs + mkdir -p backend/logs + mkdir -p backend/screenshots + mkdir -p backend/data + chmod 755 deploy/ssl deploy/logs backend/logs backend/screenshots backend/data +} + +# 生成 SSL 证书（自签名） +generate_ssl_cert() { + if [ ! -f "deploy/ssl/cert.pem" ] || [ ! -f "deploy/ssl/key.pem" ]; then + print_info "生成自签名 SSL 证书..." + openssl req -x509 -newkey rsa:4096 -keyout deploy/ssl/key.pem -out deploy/ssl/cert.pem -days 365 -nodes -subj "/C=CN/ST=Beijing/L=Beijing/O=Apple Exchange/CN=localhost" 2>/dev/null || { + print_warning "无法生成 SSL 证书，请手动安装或使用 HTTP" + } + fi +} + +# 清理旧容器和镜像 +cleanup() { + print_info "清理旧容器和镜像..." + docker-compose -f deploy/docker-compose.combined.yml down --remove-orphans 2>/dev/null || true + docker system prune -f 2>/dev/null || true +} + +# 构建和启动服务 +start_services() { + local env="$1" + print_info "启动 $env 环境服务..." + + case $env in + "dev") + print_info "启动开发环境..." + docker-compose -f deploy/docker-compose.combined.yml --profile dev up -d + ;; + "prod") + print_info "启动生产环境..." + docker-compose -f deploy/docker-compose.combined.yml --profile production up -d + ;; + "monitoring") + print_info "启动包含监控的环境..." + docker-compose -f deploy/docker-compose.combined.yml --profile monitoring --profile production up -d + ;; + *) + print_error "未知的环境: $env" + echo "可用环境: dev, prod, monitoring" + exit 1 + ;; + esac +} + +# 等待服务启动 +wait_for_services() { + print_info "等待服务启动..." + sleep 30 + + # 检查前端服务 + if curl -f http://localhost:3000/ >/dev/null 2>&1; then + print_success "前端服务已启动: http://localhost:3000" + else + print_warning "前端服务启动失败或仍在启动中" + fi + + # 检查后端 API + if curl -f http://localhost:8000/api/v1/health/liveness >/dev/null 2>&1; then + print_success "后端 API 已启动: http://localhost:8000" + print_success "API 文档: http://localhost:8000/docs" + else + print_warning "后端 API 启动失败或仍在启动中" + fi + + # 检查数据库 + if docker exec apple-exchange-db pg_isready -U postgres >/dev/null 2>&1; then + print_success "数据库已启动" + else + print_warning "数据库启动失败或仍在启动中" + fi + + # 检查 Redis + if docker exec apple-exchange-redis redis-cli ping >/dev/null 2>&1; then + print_success "Redis 已启动" + else + print_warning "Redis 启动失败或仍在启动中" + fi +} + +# 显示服务状态 +show_status() { + print_info "服务状态:" + docker-compose -f deploy/docker-compose.combined.yml ps +} + +# 显示访问信息 +show_access_info() { + echo "" + print_success "部署完成！" + echo "" + echo "=== 访问地址 ===" + echo "前端应用: http://localhost:3000" + echo "后端 API: http://localhost:8000" + echo "API 文档: http://localhost:8000/docs" + echo "" + echo "=== 监控服务 (如果启用) ===" + echo "Celery Flower: http://localhost:5555" + echo "" + echo "=== 数据库访问 ===" + echo "PostgreSQL: localhost:5432" + echo "Redis: localhost:6379" + echo "" + echo "=== 管理命令 ===" + echo "查看日志: docker-compose -f deploy/docker-compose.combined.yml logs -f [service]" + echo "停止服务: docker-compose -f deploy/docker-compose.combined.yml down" + echo "重启服务: docker-compose -f deploy/docker-compose.combined.yml restart [service]" + echo "" +} + +# 主函数 +main() { + local env="${1:-dev}" + + print_info "Apple Gift Card Exchange Platform 部署脚本" + print_info "环境: $env" + echo "" + + check_docker + check_env_files + create_directories + generate_ssl_cert + + if [ "$2" = "--cleanup" ]; then + cleanup + fi + + start_services "$env" + wait_for_services + show_status + show_access_info +} + +# 解析命令行参数 +if [ $# -eq 0 ]; then + main "dev" +else + main "$1" "$2" +fi \ No newline at end of file diff --git a/backend/deploy/docker-compose.yml b/deploy/docker-compose.combined.yml similarity index 69% rename from backend/deploy/docker-compose.yml rename to deploy/docker-compose.combined.yml index 426cbfd..ce5ac82 100644 --- a/backend/deploy/docker-compose.yml +++ b/deploy/docker-compose.combined.yml @@ -1,11 +1,56 @@ +# Apple Gift Card Exchange Platform - Combined Docker Compose +# 包含前端和后端的完整部署配置 + services: - # FastAPI API服务 + # ===== 前端服务 ===== + frontend: + build: + context: ../frontend + dockerfile: Dockerfile + container_name: apple-exchange-frontend + ports: + - "3000:8080" + environment: + - NODE_ENV=production + - NEXT_PUBLIC_ENV=production + - NEXT_PUBLIC_API_URL=http://api:8000/api + - NEXT_PUBLIC_TELEMETRY_ENDPOINT=http://otel-collector:4318/v1/traces + networks: + - app-network + depends_on: + - api + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + deploy: + mode: replicated + replicas: 1 + update_config: + parallelism: 1 + delay: 10s + failure_action: rollback + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + cpus: '0.2' + memory: 256M + + # ===== 后端 API 服务 ===== api: build: - context: .. - dockerfile: deploy/Dockerfile - args: - - SERVICE_TYPE=api + context: ../backend + dockerfile: Dockerfile + container_name: apple-exchange-api environment: - SERVICE_TYPE=api - ENVIRONMENT=production @@ -13,7 +58,6 @@ services: - REDIS_URL=redis://redis:6379/0 - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/1 - - OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:14250 - WORKERS=4 - SCREENSHOT_DIR=/app/screenshots - LOG_DIR=/app/logs @@ -22,8 +66,13 @@ services: - data:/app/data - screenshots:/app/screenshots - shared_storage:/app/shared + ports: + - "8000:8000" networks: - app-network + depends_on: + - db + - redis healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/health/liveness"] interval: 30s @@ -32,7 +81,7 @@ services: start_period: 40s deploy: mode: replicated - replicas: 2 + replicas: 1 update_config: parallelism: 1 delay: 10s @@ -45,9 +94,6 @@ services: delay: 5s max_attempts: 3 window: 120s - placement: - constraints: - - node.role == worker resources: limits: cpus: '1.0' @@ -56,13 +102,12 @@ services: cpus: '0.5' memory: 512M - # Celery Worker服务 (可扩展多个副本) + # ===== Celery Worker 服务 ===== worker: build: - context: .. - dockerfile: deploy/Dockerfile - args: - - SERVICE_TYPE=worker + context: ../backend + dockerfile: Dockerfile.worker + container_name: apple-exchange-worker environment: - SERVICE_TYPE=worker - ENVIRONMENT=production @@ -84,6 +129,9 @@ services: - playwright_browsers:/app/playwright-browsers networks: - app-network + depends_on: + - db + - redis healthcheck: test: ["CMD", "python", "-c", "from app.core.celery_app import get_celery_app; app = get_celery_app(); print('Worker healthy')"] interval: 60s @@ -92,7 +140,7 @@ services: start_period: 60s deploy: mode: replicated - replicas: 4 + replicas: 1 update_config: parallelism: 2 delay: 10s @@ -105,9 +153,6 @@ services: delay: 5s max_attempts: 3 window: 120s - placement: - constraints: - - node.role == worker resources: limits: cpus: '2.0' @@ -116,13 +161,12 @@ services: cpus: '1.0' memory: 1G - # Celery Beat调度服务 + # ===== Celery Beat 调度服务 ===== beat: build: - context: .. - dockerfile: deploy/Dockerfile - args: - - SERVICE_TYPE=beat + context: ../backend + dockerfile: Dockerfile + container_name: apple-exchange-beat environment: - SERVICE_TYPE=beat - ENVIRONMENT=production @@ -135,6 +179,15 @@ services: - data:/app/data networks: - app-network + depends_on: + - db + - redis + healthcheck: + test: ["CMD", "python", "-c", "import sys; sys.exit(0)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s deploy: mode: replicated replicas: 1 @@ -147,9 +200,6 @@ services: delay: 5s max_attempts: 3 window: 120s - placement: - constraints: - - node.role == manager resources: limits: cpus: '0.5' @@ -158,55 +208,16 @@ services: cpus: '0.2' memory: 256M - # Celery Flower监控服务 - flower: - build: - context: .. - dockerfile: deploy/Dockerfile - args: - - SERVICE_TYPE=flower - ports: - - "5555:5555" - environment: - - SERVICE_TYPE=flower - - ENVIRONMENT=production - - REDIS_URL=redis://redis:6379/0 - - CELERY_BROKER_URL=redis://redis:6379/0 - networks: - - app-network - deploy: - mode: replicated - replicas: 1 - update_config: - parallelism: 1 - delay: 10s - failure_action: rollback - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - window: 120s - placement: - constraints: - - node.role == manager - resources: - limits: - cpus: '0.5' - memory: 512M - reservations: - cpus: '0.2' - memory: 256M - - # PostgreSQL数据库 + # ===== PostgreSQL 数据库 ===== db: image: postgres:17-alpine + container_name: apple-exchange-db environment: - POSTGRES_DB=apple_exchange - POSTGRES_USER=postgres - POSTGRES_PASSWORD=password volumes: - postgres_data:/var/lib/postgresql/data - - ./init.sql:/docker-entrypoint-initdb.d/init.sql ports: - "5432:5432" networks: @@ -228,9 +239,6 @@ services: delay: 5s max_attempts: 3 window: 120s - placement: - constraints: - - node.role == manager resources: limits: cpus: '1.0' @@ -239,12 +247,13 @@ services: cpus: '0.5' memory: 512M - # Redis缓存和消息代理 + # ===== Redis 缓存和消息代理 ===== redis: image: redis:7-alpine + container_name: apple-exchange-redis volumes: - redis_data:/data - - ./redis.conf:/usr/local/etc/redis/redis.conf + - ./deploy/redis.conf:/usr/local/etc/redis/redis.conf command: redis-server /usr/local/etc/redis/redis.conf ports: - "6379:6379" @@ -267,9 +276,6 @@ services: delay: 5s max_attempts: 3 window: 120s - placement: - constraints: - - node.role == manager resources: limits: cpus: '0.5' @@ -278,24 +284,69 @@ services: cpus: '0.2' memory: 256M + # ===== 可选：Celery Flower 监控服务 ===== + flower: + build: + context: ../backend + dockerfile: Dockerfile + container_name: apple-exchange-flower + environment: + - SERVICE_TYPE=flower + - ENVIRONMENT=production + - DATABASE_URL=postgresql+asyncpg://postgres:password@db:5432/apple_exchange + - REDIS_URL=redis://redis:6379/0 + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/1 + ports: + - "5555:5555" + networks: + - app-network + depends_on: + - db + - redis + profiles: + - monitoring + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + cpus: '0.2' + memory: 256M + +# ===== 数据卷 ===== volumes: postgres_data: driver: local + name: apple-exchange-postgres-data redis_data: driver: local + name: apple-exchange-redis-data shared_storage: driver: local + name: apple-exchange-shared-storage playwright_browsers: driver: local + name: apple-exchange-playwright-browsers logs: driver: local + name: apple-exchange-logs data: driver: local + name: apple-exchange-data screenshots: driver: local + name: apple-exchange-screenshots +# ===== 网络 ===== networks: app-network: - driver: overlay - attachable: true + driver: bridge + name: apple-exchange-network + ipam: + config: + - subnet: 172.20.0.0/16 \ No newline at end of file diff --git a/deploy/redis.conf b/deploy/redis.conf new file mode 100644 index 0000000..50fc8d3 --- /dev/null +++ b/deploy/redis.conf @@ -0,0 +1,42 @@ +# Redis configuration for Apple Exchange Platform + +# Basic settings +port 6379 +bind 0.0.0.0 + +# Memory management +maxmemory 512mb +maxmemory-policy allkeys-lru + +# Persistence +save 900 1 +save 300 10 +save 60 10000 + +# Security +protected-mode no + +# Logging +loglevel notice + +# Network +timeout 0 +tcp-keepalive 300 + +# Clients +maxclients 10000 + +# Advanced config +rdbcompression yes +rdbchecksum yes +stop-writes-on-bgsave-error yes + +# Lua scripting +lua-time-limit 5000 + +# Slow log +slowlog-log-slower-than 10000 +slowlog-max-len 128 + +# Latency monitoring +latency-monitor-threshold 100 \ No newline at end of file diff --git a/frontend/.dockerignore b/frontend/.dockerignore index 5fb7594..cc583df 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1,16 +1,70 @@ # Dependencies -node_modules +node_modules/ +.pnpm-store/ +.npm/ +.yarn/ +.pnpm/ +bun.lockb +pnpm-lock.yaml +yarn.lock +package-lock.json npm-debug.log* yarn-debug.log* yarn-error.log* +pnpm-debug.log* # Next.js -.next -out +.next/ +out/ +.swc/ +next-env.d.ts +.vercel/ +.turbo/ # Build artifacts -dist -build +dist/ +build/ +public/_next/ +storybook-static/ +tsconfig.tsbuildinfo + +# Development files +.env +.env.* +!.env.example +.env.local +.env.development +.env.test +.env.production + +# Docker and deployment files +deploy/ +docker-compose*.yml +.docker/ + +# Git +.git/ +.gitignore + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Test and coverage +coverage/ +.nyc_output/ +cypress/videos/ +cypress/screenshots/ +.cypress-cache/ + +# Temporary files +*.log +.temp/ +.cache/ # Environment variables .env diff --git a/frontend/Dockerfile b/frontend/Dockerfile index e74b16a..a4cf560 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -74,6 +74,20 @@ server { location / { try_files \$uri \$uri/ \$uri.html /index.html; } + + # Handle Next.js asset paths + location /_next/static/ { + alias /usr/share/nginx/html/_next/static/; + expires 1y; + add_header Cache-Control "public, immutable"; + } + + # Handle static assets + location /static/ { + alias /usr/share/nginx/html/static/; + expires 1y; + add_header Cache-Control "public, immutable"; + } } EOF diff --git a/frontend/deploy/.dockerignore b/frontend/deploy/.dockerignore deleted file mode 100644 index 11ab00c..0000000 --- a/frontend/deploy/.dockerignore +++ /dev/null @@ -1,37 +0,0 @@ -# 依赖 -node_modules -npm-debug.log -yarn-debug.log -yarn-error.log - -# 构建产物 -.next -out -dist -build - -# 开发环境 -.env.local -.env.development.local -.env.test.local -.env.production.local - -# 版本控制 -.git -.github -.gitignore - -# IDE配置 -.idea -.vscode -*.sublime-project -*.sublime-workspace - -# 其他 -README.md -LICENSE -.DS_Store -*.pem -coverage -.vercel -.turbo \ No newline at end of file diff --git a/frontend/deploy/Dockerfile b/frontend/deploy/Dockerfile deleted file mode 100644 index 2983368..0000000 --- a/frontend/deploy/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -FROM node:24-alpine AS base - -# 安装依赖阶段 -FROM base AS deps -WORKDIR /app - -# 复制package.json和package-lock.json -COPY package.json package-lock.json ./ - -# 安装依赖 -RUN npm ci - -# 构建阶段 -FROM base AS builder -WORKDIR /app - -# 复制依赖 -COPY --from=deps /app/node_modules ./node_modules -COPY . . - -# 设置环境变量 -ARG NEXT_PUBLIC_ENV=production -ENV NEXT_PUBLIC_ENV=${NEXT_PUBLIC_ENV} -ARG NEXT_PUBLIC_API_URL=http://api:8000/api -ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} -ARG NEXT_PUBLIC_TELEMETRY_ENDPOINT=http://otel-collector:4318/v1/traces -ENV NEXT_PUBLIC_TELEMETRY_ENDPOINT=${NEXT_PUBLIC_TELEMETRY_ENDPOINT} - -# 构建应用 -RUN npm run build - -# 生产阶段 -FROM base AS runner -WORKDIR /app - -ENV NODE_ENV=production - -# 创建非root用户 -RUN addgroup --system --gid 1001 nodejs -RUN adduser --system --uid 1001 nextjs - -# 复制必要文件 -COPY --from=builder /app/public ./public -COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ -COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static - -# 切换到非root用户 -USER nextjs - -# 暴露端口 -EXPOSE 3000 - -ENV PORT 3000 -ENV HOSTNAME "0.0.0.0" - -# 启动应用 -CMD ["node", "server.js"] \ No newline at end of file diff --git a/frontend/deploy/k8s/configmap.yaml b/frontend/deploy/k8s/configmap.yaml deleted file mode 100644 index 5deb560..0000000 --- a/frontend/deploy/k8s/configmap.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: crawler-monitor-frontend-config -data: - NEXT_PUBLIC_ENV: "production" - NEXT_PUBLIC_API_URL: "http://crawler-monitor-api-service:8000/api" - NEXT_PUBLIC_TELEMETRY_ENDPOINT: "http://otel-collector:4318/v1/traces" diff --git a/frontend/deploy/k8s/deployment.yaml b/frontend/deploy/k8s/deployment.yaml deleted file mode 100644 index 5d78a2f..0000000 --- a/frontend/deploy/k8s/deployment.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: crawler-monitor-frontend - labels: - app: crawler-monitor-frontend -spec: - replicas: 2 - selector: - matchLabels: - app: crawler-monitor-frontend - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - type: RollingUpdate - template: - metadata: - labels: - app: crawler-monitor-frontend - spec: - containers: - - name: crawler-monitor-frontend - image: ${DOCKER_REGISTRY}/crawler-monitor-frontend:${IMAGE_TAG} - imagePullPolicy: Always - ports: - - containerPort: 3000 - name: http - env: - - name: NEXT_PUBLIC_ENV - value: "production" - - name: NEXT_PUBLIC_API_URL - value: "http://crawler-monitor-api-service:8000/api" - - name: NEXT_PUBLIC_TELEMETRY_ENDPOINT - value: "http://otel-collector:4318/v1/traces" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - readinessProbe: - httpGet: - path: /api/health - port: http - initialDelaySeconds: 10 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /api/health - port: http - initialDelaySeconds: 20 - periodSeconds: 15 - imagePullSecrets: - - name: regcred diff --git a/frontend/deploy/k8s/ingress.yaml b/frontend/deploy/k8s/ingress.yaml deleted file mode 100644 index 69b0ac9..0000000 --- a/frontend/deploy/k8s/ingress.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: crawler-monitor-frontend-ingress - annotations: - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$1 - nginx.ingress.kubernetes.io/proxy-body-size: "50m" - nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" - nginx.ingress.kubernetes.io/proxy-send-timeout: "300" - nginx.ingress.kubernetes.io/proxy-read-timeout: "300" -spec: - rules: - - host: crawler-monitor.example.com - http: - paths: - - path: /(.*) - pathType: Prefix - backend: - service: - name: crawler-monitor-frontend-service - port: - number: 80 diff --git a/frontend/deploy/k8s/service.yaml b/frontend/deploy/k8s/service.yaml deleted file mode 100644 index 3941d1c..0000000 --- a/frontend/deploy/k8s/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: crawler-monitor-frontend-service - labels: - app: crawler-monitor-frontend -spec: - selector: - app: crawler-monitor-frontend - ports: - - port: 80 - targetPort: 3000 - protocol: TCP - name: http - type: ClusterIP diff --git a/frontend/deploy/nginx.conf b/frontend/deploy/nginx.conf deleted file mode 100644 index f9765bc..0000000 --- a/frontend/deploy/nginx.conf +++ /dev/null @@ -1,128 +0,0 @@ -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log notice; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - - keepalive_timeout 65; - - gzip on; - gzip_disable "msie6"; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_buffers 16 8k; - gzip_http_version 1.1; - gzip_min_length 256; - gzip_types - application/atom+xml - application/geo+json - application/javascript - application/x-javascript - application/json - application/ld+json - application/manifest+json - application/rdf+xml - application/rss+xml - application/xhtml+xml - application/xml - font/eot - font/otf - font/ttf - image/svg+xml - text/css - text/javascript - text/plain - text/xml; - - server { - listen 80; - server_name localhost; - root /usr/share/nginx/html; - index index.html; - - # 安全相关头部 - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - add_header X-Frame-Options SAMEORIGIN; - add_header Referrer-Policy strict-origin-when-cross-origin; - add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self' http://api:8000 http://otel-collector:4318;"; - - # 静态资源缓存 - location /_next/static/ { - alias /usr/share/nginx/html/_next/static/; - expires 365d; - add_header Cache-Control "public, max-age=31536000, immutable"; - } - - location /static/ { - expires 365d; - add_header Cache-Control "public, max-age=31536000, immutable"; - } - - # 代理到Next.js应用 - location / { - proxy_pass http://localhost:3000; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # 代理到后端API - location /api/ { - proxy_pass http://api:8000/api/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # 健康检查 - location /health { - access_log off; - return 200 'OK'; - add_header Content-Type text/plain; - } - - # 404错误处理 - error_page 404 /404.html; - location = /404.html { - root /usr/share/nginx/html; - internal; - } - - # 50x错误处理 - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - internal; - } - } -} \ No newline at end of file