diff --git a/database/database.go b/database/database.go index a644dd3..51afd5a 100644 --- a/database/database.go +++ b/database/database.go @@ -61,6 +61,11 @@ func Initialize(dataDir string) error { sqlDB.SetMaxIdleConns(1) sqlDB.SetConnMaxLifetime(time.Hour) + // 在自动迁移之前清理旧的CHECK约束 + if err := cleanupOldConstraints(); err != nil { + return fmt.Errorf("failed to cleanup old constraints: %w", err) + } + // 自动迁移数据库结构 if err := autoMigrate(); err != nil { return fmt.Errorf("failed to migrate database: %w", err) @@ -80,6 +85,80 @@ func autoMigrate() error { ) } +// cleanupOldConstraints 清理旧的CHECK约束 +func cleanupOldConstraints() error { + // 检查data_sources表是否存在且包含CHECK约束 + var count int64 + err := DB.Raw("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='data_sources'").Scan(&count).Error + if err != nil { + return err + } + + // 如果表不存在，直接返回 + if count == 0 { + log.Println("data_sources表不存在，跳过约束清理") + return nil + } + + // 检查是否有CHECK约束 + var constraintCount int64 + err = DB.Raw("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='data_sources' AND sql LIKE '%CHECK%'").Scan(&constraintCount).Error + if err != nil { + return err + } + + // 如果没有CHECK约束，直接返回 + if constraintCount == 0 { + log.Println("data_sources表没有CHECK约束，跳过清理") + return nil + } + + log.Println("检测到旧的CHECK约束，开始清理...") + + // 重建表，去掉CHECK约束 + // 1. 创建新表（不包含CHECK约束） + createNewTableSQL := ` + CREATE TABLE data_sources_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + endpoint_id INTEGER NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL, + config TEXT NOT NULL, + is_active BOOLEAN DEFAULT true, + last_sync DATETIME, + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME + )` + + if err := DB.Exec(createNewTableSQL).Error; err != nil { + return fmt.Errorf("创建新表失败: %w", err) + } + + // 2. 复制数据 + copyDataSQL := ` + INSERT INTO data_sources_new (id, endpoint_id, name, type, config, is_active, last_sync, created_at, updated_at, deleted_at) + SELECT id, endpoint_id, name, type, config, is_active, last_sync, created_at, updated_at, deleted_at + FROM data_sources` + + if err := DB.Exec(copyDataSQL).Error; err != nil { + return fmt.Errorf("复制数据失败: %w", err) + } + + // 3. 删除旧表 + if err := DB.Exec("DROP TABLE data_sources").Error; err != nil { + return fmt.Errorf("删除旧表失败: %w", err) + } + + // 4. 重命名新表 + if err := DB.Exec("ALTER TABLE data_sources_new RENAME TO data_sources").Error; err != nil { + return fmt.Errorf("重命名表失败: %w", err) + } + + log.Println("旧的CHECK约束清理完成") + return nil +} + // Close 关闭数据库连接 func Close() error { if DB != nil { diff --git a/go.mod b/go.mod index 4f89f1b..d53279f 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,30 @@ go 1.23.0 toolchain go1.23.1 require ( + github.com/aws/aws-sdk-go-v2 v1.36.4 + github.com/aws/aws-sdk-go-v2/config v1.29.16 + github.com/aws/aws-sdk-go-v2/credentials v1.17.69 + github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 github.com/glebarez/sqlite v1.11.0 golang.org/x/time v0.12.0 gorm.io/gorm v1.30.0 ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/google/uuid v1.5.0 // indirect diff --git a/go.sum b/go.sum index 522393b..c434a54 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,39 @@ +github.com/aws/aws-sdk-go-v2 v1.36.4 h1:GySzjhVvx0ERP6eyfAbAuAXLtAda5TEy19E5q5W8I9E= +github.com/aws/aws-sdk-go-v2 v1.36.4/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= +github.com/aws/aws-sdk-go-v2/config v1.29.16 h1:XkruGnXX1nEZ+Nyo9v84TzsX+nj86icbFAeust6uo8A= +github.com/aws/aws-sdk-go-v2/config v1.29.16/go.mod h1:uCW7PNjGwZ5cOGZ5jr8vCWrYkGIhPoTNV23Q/tpHKzg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.69 h1:8B8ZQboRc3uaIKjshve/XlvJ570R7BKNy3gftSbS178= +github.com/aws/aws-sdk-go-v2/credentials v1.17.69/go.mod h1:gPME6I8grR1jCqBFEGthULiolzf/Sexq/Wy42ibKK9c= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 h1:oQWSGexYasNpYp4epLGZxxjsDo8BMBh6iNWkTXQvkwk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31/go.mod h1:nc332eGUU+djP3vrMI6blS0woaCfHTe3KiSQUVTMRq0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 h1:o1v1VFfPcDVlK3ll1L5xHsaQAFdNtZ5GXnNR7SwueC4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35/go.mod h1:rZUQNYMNG+8uZxz9FOerQJ+FceCiodXvixpeRtdESrU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 h1:R5b82ubO2NntENm3SAm0ADME+H630HomNJdgv+yZ3xw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35/go.mod h1:FuA+nmgMRfkzVKYDNEqQadvEMxtxl9+RLT9ribCwEMs= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 h1:th/m+Q18CkajTw1iqx2cKkLCij/uz8NMwJFPK91p2ug= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35/go.mod h1:dkJuf0a1Bc8HAA0Zm2MoTGm/WDC18Td9vSbrQ1+VqE8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 h1:VHPZakq2L7w+RLzV54LmQavbvheFaR2u1NomJRSEfcU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3/go.mod h1:DX1e/lkbsAt0MkY3NgLYuH4jQvRfw8MYxTe9feR7aXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 h1:/ldKrPPXTC421bTNWrUIpq3CxwHwRI/kpc+jPUTJocM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16/go.mod h1:5vkf/Ws0/wgIMJDQbjI4p2op86hNW6Hie5QtebrDgT8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 h1:2HuI7vWKhFWsBhIr2Zq8KfFZT6xqaId2XXnXZjkbEuc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16/go.mod h1:BrwWnsfbFtFeRjdx0iM1ymvlqDX1Oz68JsQaibX/wG8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 h1:T6Wu+8E2LeTUqzqQ/Bh1EoFNj1u4jUyveMgmTlu9fDU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2/go.mod h1:chSY8zfqmS0OnhZoO/hpPx/BHfAIL80m77HwhRLYScY= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 h1:EU58LP8ozQDVroOEyAfcq0cGc5R/FTZjVoYJ6tvby3w= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.4/go.mod h1:CrtOgCcysxMvrCoHnvNAD7PHWclmoFG78Q2xLK0KKcs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 h1:XB4z0hbQtpmBnb1FQYvKaCM7UsS6Y/u8jVBwIUGeCTk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2/go.mod h1:hwRpqkRxnQ58J9blRDrB4IanlXCpcKmsC83EhG77upg= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 h1:nyLjs8sYJShFYj6aiyjCBI3EcLn1udWrQTjEF+SOXB0= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.21/go.mod h1:EhdxtZ+g84MSGrSrHzZiUm9PYiZkrADNja15wtRJSJo= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= diff --git a/initapp/init.go b/initapp/init.go index 9107ec2..019ff45 100644 --- a/initapp/init.go +++ b/initapp/init.go @@ -32,19 +32,34 @@ func InitData() error { // 4. 统计需要预加载的数据源 var activeDataSources []model.DataSource + var totalDataSources, disabledDataSources, apiDataSources int + for _, endpoint := range endpoints { if !endpoint.IsActive { continue } for _, ds := range endpoint.DataSources { - if ds.IsActive && ds.Type != "api_get" && ds.Type != "api_post" { - // API类型的数据源不需要预加载，使用实时请求 - activeDataSources = append(activeDataSources, ds) + totalDataSources++ + + if !ds.IsActive { + disabledDataSources++ + log.Printf("跳过禁用的数据源: %s (ID: %d)", ds.Name, ds.ID) + continue } + + if ds.Type == "api_get" || ds.Type == "api_post" { + apiDataSources++ + log.Printf("跳过API类型数据源: %s (ID: %d, 类型: %s) - 使用实时请求", ds.Name, ds.ID, ds.Type) + continue + } + + // 需要预加载的数据源 + activeDataSources = append(activeDataSources, ds) } } - log.Printf("发现 %d 个端点，%d 个需要预加载的数据源", len(endpoints), len(activeDataSources)) + log.Printf("发现 %d 个端点，总共 %d 个数据源", len(endpoints), totalDataSources) + log.Printf("其中: 禁用 %d 个，API类型 %d 个，需要预加载 %d 个", disabledDataSources, apiDataSources, len(activeDataSources)) if len(activeDataSources) == 0 { log.Println("✓ 没有需要预加载的数据源") diff --git a/model/api_endpoint.go b/model/api_endpoint.go index 2353763..f131724 100644 --- a/model/api_endpoint.go +++ b/model/api_endpoint.go @@ -29,7 +29,7 @@ type DataSource struct { ID uint `json:"id" gorm:"primaryKey"` EndpointID uint `json:"endpoint_id" gorm:"not null;index"` Name string `json:"name" gorm:"not null"` - Type string `json:"type" gorm:"not null;check:type IN ('lankong', 'manual', 'api_get', 'api_post', 'endpoint')"` + Type string `json:"type" gorm:"not null"` Config string `json:"config" gorm:"not null"` IsActive bool `json:"is_active" gorm:"default:true"` LastSync *time.Time `json:"last_sync,omitempty"` @@ -80,6 +80,9 @@ type DataSourceConfig struct { // 端点配置 EndpointConfig *EndpointConfig `json:"endpoint_config,omitempty"` + + // S3配置 + S3Config *S3Config `json:"s3_config,omitempty"` } type LankongConfig struct { @@ -103,3 +106,26 @@ type APIConfig struct { type EndpointConfig struct { EndpointIDs []uint `json:"endpoint_ids"` // 选中的端点ID列表 } + +// S3Config S3通用协议配置 +type S3Config struct { + // 基础配置 + Endpoint string `json:"endpoint"` // S3端点地址 + BucketName string `json:"bucket_name"` // 存储桶名称 + Region string `json:"region"` // 地区 + AccessKeyID string `json:"access_key_id"` // 访问密钥ID + SecretAccessKey string `json:"secret_access_key"` // 访问密钥 + + // 高级配置 + ListObjectsVersion string `json:"list_objects_version"` // 列出对象版本 (v1/v2)，默认v2 + UsePathStyle bool `json:"use_path_style"` // 是否使用path style + RemoveBucket bool `json:"remove_bucket"` // 是否从路径中删除bucket名称 + + // 自定义域名配置 + CustomDomain string `json:"custom_domain"` // 自定义访问域名，支持路径 + + // 文件过滤配置 + FolderPath string `json:"folder_path"` // 提取的文件夹路径，例如: /img + IncludeSubfolders bool `json:"include_subfolders"` // 是否提取所有子文件夹 + FileExtensions []string `json:"file_extensions"` // 提取的文件格式后缀，支持正则匹配 +} diff --git a/readme.md b/readme.md index 2496f80..cd820c7 100644 --- a/readme.md +++ b/readme.md @@ -4,7 +4,7 @@ ## 功能特性 -- 🎯 支持多种数据源：兰空图床API、手动配置、通用API接口 +- 🎯 支持多种数据源：兰空图床API, s3协议对象存储, 手动配置, 通用API接口(GET/POST) - 🔐 OAuth2.0 管理后台登录（CZL Connect） - 💾 SQLite数据库存储 - ⚡ 内存缓存机制 @@ -12,107 +12,9 @@ - 📝 可配置首页内容 - 🎨 现代化Web管理界面 -## 环境变量配置 +## 部署and讨论 -复制 `env.example` 为 `.env` 并配置以下环境变量： - -```bash -# 服务器配置 -PORT=:5003 # 服务端口 -READ_TIMEOUT=30s # 读取超时 -WRITE_TIMEOUT=30s # 写入超时 -MAX_HEADER_BYTES=1048576 # 最大请求头大小 - -# 数据存储目录 -DATA_DIR=./data # 数据存储目录 - -# OAuth2.0 配置 (必需) -OAUTH_CLIENT_ID=your-oauth-client-id # CZL Connect 客户端ID -OAUTH_CLIENT_SECRET=your-oauth-client-secret # CZL Connect 客户端密钥 -``` - -## 快速开始 - -1. 克隆项目 -```bash -git clone -cd random-api-go -``` - -2. 配置环境变量 -```bash -cp env.example .env -# 编辑 .env 文件，填入正确的 OAuth 配置 -``` - -3. 运行服务 -```bash -go run main.go -``` - -4. 访问服务 -- 首页: http://localhost:5003 -- 管理后台: http://localhost:5003/admin - -## OAuth2.0 配置 - -本项目使用 CZL Connect 作为 OAuth2.0 提供商： - -- 授权端点: https://connect.czl.net/oauth2/authorize -- 令牌端点: https://connect.czl.net/api/oauth2/token -- 用户信息端点: https://connect.czl.net/api/oauth2/userinfo - -请在 CZL Connect 中注册应用并获取 `client_id` 和 `client_secret`。 - -## API 端点 - -### 公开API -- `GET /` - 首页 -- `GET /{endpoint}` - 随机API端点 - -### 管理API -- `GET /admin/api/oauth-config` - 获取OAuth配置 -- `POST /admin/api/oauth-verify` - 验证OAuth授权码 -- `GET /admin/api/endpoints` - 列出所有端点 -- `POST /admin/api/endpoints/` - 创建端点 -- `GET /admin/api/endpoints/{id}` - 获取端点详情 -- `PUT /admin/api/endpoints/{id}` - 更新端点 -- `DELETE /admin/api/endpoints/{id}` - 删除端点 -- `POST /admin/api/data-sources` - 创建数据源 -- `GET /admin/api/url-replace-rules` - 列出URL替换规则 -- `POST /admin/api/url-replace-rules/` - 创建URL替换规则 -- `GET /admin/api/home-config` - 获取首页配置 -- `PUT /admin/api/home-config/` - 更新首页配置 - -## 数据源类型 - -1. **兰空图床 (lankong)**: 从兰空图床API获取图片 -2. **手动配置 (manual)**: 手动配置的URL列表 -3. **API GET (api_get)**: 从GET接口获取数据 -4. **API POST (api_post)**: 从POST接口获取数据 - -## 部署 - -### Docker 部署 - -```dockerfile -FROM golang:1.21-alpine AS builder -WORKDIR /app -COPY . . -RUN go build -o random-api-server main.go - -FROM alpine:latest -RUN apk --no-cache add ca-certificates -WORKDIR /root/ -COPY --from=builder /app/random-api-server . -COPY --from=builder /app/web ./web -EXPOSE 5003 -CMD ["./random-api-server"] -``` - -### 环境变量部署 - -确保在生产环境中正确设置所有必需的环境变量，特别是OAuth配置。 + ## 许可证 diff --git a/service/data_source_fetcher.go b/service/data_source_fetcher.go index 9e6b87e..18ad1c4 100644 --- a/service/data_source_fetcher.go +++ b/service/data_source_fetcher.go @@ -16,6 +16,7 @@ type DataSourceFetcher struct { cacheManager *CacheManager lankongFetcher *LankongFetcher apiFetcher *APIFetcher + s3Fetcher *S3Fetcher } // NewDataSourceFetcher 创建数据源获取器 @@ -38,6 +39,7 @@ func NewDataSourceFetcher(cacheManager *CacheManager) *DataSourceFetcher { cacheManager: cacheManager, lankongFetcher: lankongFetcher, apiFetcher: NewAPIFetcher(), + s3Fetcher: NewS3Fetcher(), } } @@ -86,6 +88,8 @@ func (dsf *DataSourceFetcher) FetchURLs(dataSource *model.DataSource) ([]string, urls, err = dsf.fetchManualURLs(dataSource) case "endpoint": urls, err = dsf.fetchEndpointURLs(dataSource) + case "s3": + urls, err = dsf.fetchS3URLs(dataSource) default: return nil, fmt.Errorf("unsupported data source type: %s", dataSource.Type) } @@ -180,6 +184,16 @@ func (dsf *DataSourceFetcher) fetchEndpointURLs(dataSource *model.DataSource) ([ return urls, nil } +// fetchS3URLs 获取S3存储桶URL +func (dsf *DataSourceFetcher) fetchS3URLs(dataSource *model.DataSource) ([]string, error) { + var config model.S3Config + if err := json.Unmarshal([]byte(dataSource.Config), &config); err != nil { + return nil, fmt.Errorf("invalid S3 config: %w", err) + } + + return dsf.s3Fetcher.FetchURLs(&config) +} + // updateDataSourceSyncTime 更新数据源的同步时间 func (dsf *DataSourceFetcher) updateDataSourceSyncTime(dataSource *model.DataSource) error { // 这里需要导入database包来更新数据库 diff --git a/service/endpoint_service.go b/service/endpoint_service.go index d043d89..54ceb53 100644 --- a/service/endpoint_service.go +++ b/service/endpoint_service.go @@ -22,6 +22,26 @@ type EndpointService struct { var endpointService *EndpointService var once sync.Once +// 支持的数据源类型列表 +var supportedDataSourceTypes = []string{ + "lankong", + "manual", + "api_get", + "api_post", + "endpoint", + "s3", +} + +// validateDataSourceType 验证数据源类型 +func validateDataSourceType(dataSourceType string) error { + for _, supportedType := range supportedDataSourceTypes { + if dataSourceType == supportedType { + return nil + } + } + return fmt.Errorf("unsupported data source type: %s, supported types: %v", dataSourceType, supportedDataSourceTypes) +} + // GetEndpointService 获取端点服务单例 func GetEndpointService() *EndpointService { once.Do(func() { @@ -287,6 +307,11 @@ func (s *EndpointService) applyURLReplaceRules(url, endpointURL string) string { // CreateDataSource 创建数据源 func (s *EndpointService) CreateDataSource(dataSource *model.DataSource) error { + // 验证数据源类型 + if err := validateDataSourceType(dataSource.Type); err != nil { + return err + } + if err := database.DB.Create(dataSource).Error; err != nil { return fmt.Errorf("failed to create data source: %w", err) } @@ -304,6 +329,11 @@ func (s *EndpointService) CreateDataSource(dataSource *model.DataSource) error { // UpdateDataSource 更新数据源 func (s *EndpointService) UpdateDataSource(dataSource *model.DataSource) error { + // 验证数据源类型 + if err := validateDataSourceType(dataSource.Type); err != nil { + return err + } + if err := database.DB.Save(dataSource).Error; err != nil { return fmt.Errorf("failed to update data source: %w", err) } diff --git a/service/preloader.go b/service/preloader.go index 174ccdd..cfd2962 100644 --- a/service/preloader.go +++ b/service/preloader.go @@ -74,6 +74,12 @@ func (p *Preloader) ResumePeriodicRefresh() { // PreloadDataSourceOnSave 在保存数据源时预加载数据 func (p *Preloader) PreloadDataSourceOnSave(dataSource *model.DataSource) { + // 检查数据源是否处于活跃状态 + if !dataSource.IsActive { + log.Printf("数据源 %d 已禁用，跳过预加载", dataSource.ID) + return + } + // API类型的数据源不需要预加载，使用实时请求 if dataSource.Type == "api_get" || dataSource.Type == "api_post" { log.Printf("API数据源 %d (%s) 使用实时请求，跳过预加载", dataSource.ID, dataSource.Type) @@ -135,6 +141,18 @@ func (p *Preloader) RefreshDataSource(dataSourceID uint) error { return err } + // 检查数据源是否处于活跃状态 + if !dataSource.IsActive { + log.Printf("数据源 %d 已禁用，跳过刷新", dataSourceID) + return nil + } + + // API类型的数据源不需要预加载，使用实时请求 + if dataSource.Type == "api_get" || dataSource.Type == "api_post" { + log.Printf("API数据源 %d (%s) 使用实时请求，跳过刷新", dataSource.ID, dataSource.Type) + return nil + } + log.Printf("手动刷新数据源 %d", dataSourceID) return p.dataSourceFetcher.PreloadDataSource(&dataSource) } @@ -281,6 +299,8 @@ func (p *Preloader) shouldPeriodicRefresh(dataSource *model.DataSource) bool { switch dataSource.Type { case "lankong": refreshInterval = 24 * time.Hour // 兰空图床每24小时刷新一次 + case "s3": + refreshInterval = 6 * time.Hour // S3存储每6小时刷新一次 default: return false } diff --git a/service/s3_fetcher.go b/service/s3_fetcher.go new file mode 100644 index 0000000..cc8a0d9 --- /dev/null +++ b/service/s3_fetcher.go @@ -0,0 +1,330 @@ +package service + +import ( + "context" + "fmt" + "log" + "net/url" + "random-api-go/model" + "regexp" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +// S3Fetcher S3获取器 +type S3Fetcher struct { + timeout time.Duration +} + +// NewS3Fetcher 创建S3获取器 +func NewS3Fetcher() *S3Fetcher { + return &S3Fetcher{ + timeout: 30 * time.Second, + } +} + +// FetchURLs 从S3存储桶获取文件URL列表 +func (sf *S3Fetcher) FetchURLs(s3Config *model.S3Config) ([]string, error) { + if s3Config == nil { + return nil, fmt.Errorf("S3配置不能为空") + } + + // 验证必需的配置 + if s3Config.Endpoint == "" { + return nil, fmt.Errorf("S3端点地址不能为空") + } + if s3Config.BucketName == "" { + return nil, fmt.Errorf("存储桶名称不能为空") + } + if s3Config.AccessKeyID == "" { + return nil, fmt.Errorf("访问密钥ID不能为空") + } + if s3Config.SecretAccessKey == "" { + return nil, fmt.Errorf("访问密钥不能为空") + } + + // 创建S3客户端 + client, err := sf.createS3Client(s3Config) + if err != nil { + return nil, fmt.Errorf("创建S3客户端失败: %w", err) + } + + // 获取对象列表 + objects, err := sf.listObjects(client, s3Config) + if err != nil { + return nil, fmt.Errorf("获取对象列表失败: %w", err) + } + + // 过滤和转换为URL + urls := sf.convertObjectsToURLs(objects, s3Config) + + log.Printf("从S3存储桶 %s 获取到 %d 个文件URL", s3Config.BucketName, len(urls)) + return urls, nil +} + +// createS3Client 创建S3客户端 +func (sf *S3Fetcher) createS3Client(s3Config *model.S3Config) (*s3.Client, error) { + // 设置默认地区 + region := s3Config.Region + if region == "" { + region = "us-east-1" + } + + // 创建凭证 + creds := credentials.NewStaticCredentialsProvider( + s3Config.AccessKeyID, + s3Config.SecretAccessKey, + "", + ) + + // 创建配置 + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(region), + config.WithCredentialsProvider(creds), + ) + if err != nil { + return nil, fmt.Errorf("加载AWS配置失败: %w", err) + } + + // 创建S3客户端选项 + options := func(o *s3.Options) { + if s3Config.Endpoint != "" { + o.BaseEndpoint = aws.String(s3Config.Endpoint) + } + o.UsePathStyle = s3Config.UsePathStyle + } + + client := s3.NewFromConfig(cfg, options) + return client, nil +} + +// listObjects 列出存储桶中的对象 +func (sf *S3Fetcher) listObjects(client *s3.Client, s3Config *model.S3Config) ([]types.Object, error) { + ctx, cancel := context.WithTimeout(context.Background(), sf.timeout) + defer cancel() + + var allObjects []types.Object + var continuationToken *string + + // 设置前缀（文件夹路径） + prefix := strings.TrimPrefix(s3Config.FolderPath, "/") + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + // 设置分隔符（如果不包含子文件夹） + var delimiter *string + if !s3Config.IncludeSubfolders { + delimiter = aws.String("/") + } + + // 确定使用的ListObjects版本 + listVersion := s3Config.ListObjectsVersion + if listVersion == "" { + listVersion = "v2" // 默认使用v2 + } + + for { + if listVersion == "v1" { + // 使用ListObjects (v1) + input := &s3.ListObjectsInput{ + Bucket: aws.String(s3Config.BucketName), + Prefix: aws.String(prefix), + Delimiter: delimiter, + MaxKeys: aws.Int32(1000), + } + + if continuationToken != nil { + input.Marker = continuationToken + } + + result, err := client.ListObjects(ctx, input) + if err != nil { + return nil, fmt.Errorf("ListObjects失败: %w", err) + } + + allObjects = append(allObjects, result.Contents...) + + if !aws.ToBool(result.IsTruncated) { + break + } + + if len(result.Contents) > 0 { + continuationToken = result.Contents[len(result.Contents)-1].Key + } + } else { + // 使用ListObjectsV2 (v2) + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(s3Config.BucketName), + Prefix: aws.String(prefix), + Delimiter: delimiter, + MaxKeys: aws.Int32(1000), + ContinuationToken: continuationToken, + } + + result, err := client.ListObjectsV2(ctx, input) + if err != nil { + return nil, fmt.Errorf("ListObjectsV2失败: %w", err) + } + + allObjects = append(allObjects, result.Contents...) + + if !aws.ToBool(result.IsTruncated) { + break + } + + continuationToken = result.NextContinuationToken + } + } + + return allObjects, nil +} + +// convertObjectsToURLs 将S3对象转换为URL列表 +func (sf *S3Fetcher) convertObjectsToURLs(objects []types.Object, s3Config *model.S3Config) []string { + var urls []string + + // 编译文件扩展名正则表达式 + var extensionRegexes []*regexp.Regexp + for _, ext := range s3Config.FileExtensions { + if ext != "" { + // 确保扩展名以点开头 + if !strings.HasPrefix(ext, ".") { + ext = "." + ext + } + // 转义特殊字符并创建正则表达式 + pattern := regexp.QuoteMeta(ext) + "$" + if regex, err := regexp.Compile("(?i)" + pattern); err == nil { + extensionRegexes = append(extensionRegexes, regex) + } else { + log.Printf("警告: 无效的文件扩展名正则表达式 '%s': %v", ext, err) + } + } + } + + for _, obj := range objects { + if obj.Key == nil { + continue + } + + key := aws.ToString(obj.Key) + + // 跳过以/结尾的对象（文件夹） + if strings.HasSuffix(key, "/") { + continue + } + + // 如果设置了文件扩展名过滤，检查是否匹配 + if len(extensionRegexes) > 0 { + matched := false + for _, regex := range extensionRegexes { + if regex.MatchString(key) { + matched = true + break + } + } + if !matched { + continue + } + } + + // 生成URL + fileURL := sf.generateURL(key, s3Config) + if fileURL != "" { + urls = append(urls, fileURL) + } + } + + return urls +} + +// generateURL 生成文件的访问URL +func (sf *S3Fetcher) generateURL(key string, s3Config *model.S3Config) string { + // 如果设置了自定义域名 + if s3Config.CustomDomain != "" { + return sf.generateCustomDomainURL(key, s3Config) + } + + // 使用S3标准URL格式 + return sf.generateS3URL(key, s3Config) +} + +// generateCustomDomainURL 生成自定义域名URL +func (sf *S3Fetcher) generateCustomDomainURL(key string, s3Config *model.S3Config) string { + baseURL := strings.TrimSuffix(s3Config.CustomDomain, "/") + + // 处理key路径 + path := key + if s3Config.RemoveBucket { + // 如果需要移除bucket名称，并且key以bucket名称开头 + bucketPrefix := s3Config.BucketName + "/" + if strings.HasPrefix(path, bucketPrefix) { + path = strings.TrimPrefix(path, bucketPrefix) + } + } + + // 对路径进行适当的URL编码，但保留路径分隔符 + path = sf.encodeURLPath(path) + + // 确保路径以/开头 + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + return baseURL + path +} + +// generateS3URL 生成S3标准URL +func (sf *S3Fetcher) generateS3URL(key string, s3Config *model.S3Config) string { + // 对key进行适当的URL编码，但保留路径分隔符 + encodedKey := sf.encodeURLPath(key) + + if s3Config.UsePathStyle { + // Path-style URL: http://endpoint/bucket/key + endpoint := strings.TrimSuffix(s3Config.Endpoint, "/") + return fmt.Sprintf("%s/%s/%s", endpoint, s3Config.BucketName, encodedKey) + } else { + // Virtual-hosted-style URL: http://bucket.endpoint/key + endpoint := strings.TrimSuffix(s3Config.Endpoint, "/") + + // 解析endpoint以获取主机名 + if parsedURL, err := url.Parse(endpoint); err == nil { + scheme := parsedURL.Scheme + if scheme == "" { + scheme = "https" + } + host := parsedURL.Host + if host == "" { + host = parsedURL.Path + } + return fmt.Sprintf("%s://%s.%s/%s", scheme, s3Config.BucketName, host, encodedKey) + } + + // 如果解析失败，回退到path-style + return fmt.Sprintf("%s/%s/%s", endpoint, s3Config.BucketName, encodedKey) + } +} + +// encodeURLPath 对URL路径进行编码，保留路径分隔符，但编码其他特殊字符 +func (sf *S3Fetcher) encodeURLPath(path string) string { + // 分割路径为各个部分 + parts := strings.Split(path, "/") + + // 对每个部分进行URL编码 + for i, part := range parts { + if part != "" { + // 使用PathEscape对每个路径段进行编码 + // 这会将空格编码为%20，这是URL路径中的标准做法 + parts[i] = url.PathEscape(part) + } + } + + // 重新组合路径 + return strings.Join(parts, "/") +} diff --git a/web/components/admin/DataSourceConfigForm.tsx b/web/components/admin/DataSourceConfigForm.tsx index 3608a6a..63e56bc 100644 --- a/web/components/admin/DataSourceConfigForm.tsx +++ b/web/components/admin/DataSourceConfigForm.tsx @@ -11,7 +11,7 @@ import { Trash2, Plus } from 'lucide-react' import { authenticatedFetch } from '@/lib/auth' interface DataSourceConfigFormProps { - type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' + type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' | 's3' config: string onChange: (config: string) => void } @@ -40,6 +40,21 @@ interface EndpointConfig { endpoint_ids: number[] } +interface S3Config { + endpoint: string + bucket_name: string + region: string + access_key_id: string + secret_access_key: string + list_objects_version: string + use_path_style: boolean + remove_bucket: boolean + custom_domain: string + folder_path: string + include_subfolders: boolean + file_extensions: string[] +} + export default function DataSourceConfigForm({ type, config, onChange }: DataSourceConfigFormProps) { const [lankongConfig, setLankongConfig] = useState({ api_token: '', @@ -59,9 +74,25 @@ export default function DataSourceConfigForm({ type, config, onChange }: DataSou endpoint_ids: [] }) + const [s3Config, setS3Config] = useState({ + endpoint: '', + bucket_name: '', + region: '', + access_key_id: '', + secret_access_key: '', + list_objects_version: 'v2', + use_path_style: false, + remove_bucket: false, + custom_domain: '', + folder_path: '', + include_subfolders: true, + file_extensions: [] + }) + const [availableEndpoints, setAvailableEndpoints] = useState>([]) const [headerPairs, setHeaderPairs] = useState>([{key: '', value: ''}]) + const [extensionInputs, setExtensionInputs] = useState(['']) const [savedTokens, setSavedTokens] = useState([]) const [newTokenName, setNewTokenName] = useState('') @@ -127,6 +158,26 @@ export default function DataSourceConfigForm({ type, config, onChange }: DataSou setEndpointConfig({ endpoint_ids: parsed.endpoint_ids || [] }) + } else if (type === 's3') { + setS3Config({ + endpoint: parsed.endpoint || '', + bucket_name: parsed.bucket_name || '', + region: parsed.region || '', + access_key_id: parsed.access_key_id || '', + secret_access_key: parsed.secret_access_key || '', + list_objects_version: parsed.list_objects_version || 'v2', + use_path_style: parsed.use_path_style || false, + remove_bucket: parsed.remove_bucket || false, + custom_domain: parsed.custom_domain || '', + folder_path: parsed.folder_path || '', + include_subfolders: parsed.include_subfolders !== false, + file_extensions: parsed.file_extensions || [] + }) + + // 设置文件扩展名输入框 + const extensions = parsed.file_extensions || [''] + if (extensions.length === 0) extensions.push('') + setExtensionInputs(extensions) } } catch (error) { console.error('Failed to parse config:', error) @@ -257,6 +308,33 @@ export default function DataSourceConfigForm({ type, config, onChange }: DataSou updateEndpointConfig(newIds) } + // 更新S3配置 + const updateS3Config = (field: keyof S3Config, value: string | boolean | string[]) => { + const newConfig = { ...s3Config, [field]: value } + setS3Config(newConfig) + onChange(JSON.stringify(newConfig)) + } + + // 添加文件扩展名 + const addExtension = () => { + const newExtensions = [...extensionInputs, ''] + setExtensionInputs(newExtensions) + } + + // 删除文件扩展名 + const removeExtension = (index: number) => { + const newExtensions = extensionInputs.filter((_, i) => i !== index) + setExtensionInputs(newExtensions) + updateS3Config('file_extensions', newExtensions.filter(ext => ext.trim() !== '')) + } + + // 更新文件扩展名 + const updateExtension = (index: number, value: string) => { + const newExtensions = extensionInputs.map((ext, i) => i === index ? value : ext) + setExtensionInputs(newExtensions) + updateS3Config('file_extensions', newExtensions.filter(ext => ext.trim() !== '')) + } + if (type === 'manual') { return (
@@ -560,5 +638,193 @@ export default function DataSourceConfigForm({ type, config, onChange }: DataSou ) } + if (type === 's3') { + return ( +
+ {/* 基础配置 */} + + + 基础配置 + + +
+
+ + updateS3Config('endpoint', e.target.value)} + placeholder="https://s3.amazonaws.com" + /> +
+
+ + updateS3Config('bucket_name', e.target.value)} + placeholder="my-bucket" + /> +
+
+ +
+
+ + updateS3Config('region', e.target.value)} + placeholder="us-east-1" + /> +
+
+ + +
+
+ +
+
+ + updateS3Config('access_key_id', e.target.value)} + placeholder="AKIAIOSFODNN7EXAMPLE" + /> +
+
+ + updateS3Config('secret_access_key', e.target.value)} + placeholder="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + /> +
+
+ + + + {/* 高级配置 */} + + + 高级配置 + + +
+ updateS3Config('use_path_style', checked as boolean)} + /> + +
+ +
+ updateS3Config('remove_bucket', checked as boolean)} + /> + +
+ +
+ + updateS3Config('custom_domain', e.target.value)} + placeholder="https://cdn.example.com" + /> +

+ 留空使用S3标准URL，支持路径如: https://cdn.example.com/path +

+
+ + + + {/* 文件过滤配置 */} + + + 文件过滤配置 + + +
+ + updateS3Config('folder_path', e.target.value)} + placeholder="/images" + /> +

+ 指定要提取的文件夹路径，如: /images 或 /uploads/photos +

+
+ +
+ updateS3Config('include_subfolders', checked as boolean)} + /> + +
+ +
+ + {extensionInputs.map((ext, index) => ( +
+ updateExtension(index, e.target.value)} + placeholder=".jpg 或 .png" + className="flex-1" + /> + {extensionInputs.length > 1 && ( + + )} +
+ ))} + +

+ 留空表示不过滤文件格式，支持正则匹配如: .jpg, .png, .gif +

+
+ + +
+ ) + } + return null } \ No newline at end of file diff --git a/web/components/admin/DataSourceManagement.tsx b/web/components/admin/DataSourceManagement.tsx index 0b6fdf1..5748cd1 100644 --- a/web/components/admin/DataSourceManagement.tsx +++ b/web/components/admin/DataSourceManagement.tsx @@ -26,7 +26,7 @@ export default function DataSourceManagement({ const [editingDataSource, setEditingDataSource] = useState(null) const [formData, setFormData] = useState({ name: '', - type: 'manual' as 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint', + type: 'manual' as 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' | 's3', config: '', is_active: true }) @@ -185,6 +185,7 @@ export default function DataSourceManagement({ case 'api_get': return 'GET接口' case 'api_post': return 'POST接口' case 'endpoint': return '已有端点' + case 's3': return 'S3对象存储' default: return type } } @@ -245,7 +246,7 @@ export default function DataSourceManagement({
diff --git a/web/next.config.ts b/web/next.config.ts index 7187752..76ed67c 100644 --- a/web/next.config.ts +++ b/web/next.config.ts @@ -18,6 +18,14 @@ const nextConfig: NextConfig = { { source: "/api/:path*", destination: "http://localhost:5003/api/:path*", + }, + { + source: "/pic/:path*", + destination: "http://localhost:5003/pic/:path*", + }, + { + source: "/video/:path*", + destination: "http://localhost:5003/video/:path*", } ]; } diff --git a/web/types/admin.ts b/web/types/admin.ts index 3a54e49..e18243d 100644 --- a/web/types/admin.ts +++ b/web/types/admin.ts @@ -21,7 +21,7 @@ export interface DataSource { id: number endpoint_id: number name: string - type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' + type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' | 's3' config: string is_active: boolean last_sync?: string