mirror of
https://github.com/woodchen-ink/random-api-go.git
synced 2025-07-18 05:42:01 +08:00
新增对S3对象存储的支持，更新数据源配置和管理界面，优化数据源预加载逻辑，重构相关数据处理逻辑，提升系统灵活性和可维护性。
This commit is contained in:
parent
e21b5dac5f
commit
4150df03cf
@ -61,6 +61,11 @@ func Initialize(dataDir string) error {
|
||||
sqlDB.SetMaxIdleConns(1)
|
||||
sqlDB.SetConnMaxLifetime(time.Hour)
|
||||
|
||||
// 在自动迁移之前清理旧的CHECK约束
|
||||
if err := cleanupOldConstraints(); err != nil {
|
||||
return fmt.Errorf("failed to cleanup old constraints: %w", err)
|
||||
}
|
||||
|
||||
// 自动迁移数据库结构
|
||||
if err := autoMigrate(); err != nil {
|
||||
return fmt.Errorf("failed to migrate database: %w", err)
|
||||
@ -80,6 +85,80 @@ func autoMigrate() error {
|
||||
)
|
||||
}
|
||||
|
||||
// cleanupOldConstraints 清理旧的CHECK约束
|
||||
func cleanupOldConstraints() error {
|
||||
// 检查data_sources表是否存在且包含CHECK约束
|
||||
var count int64
|
||||
err := DB.Raw("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='data_sources'").Scan(&count).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 如果表不存在，直接返回
|
||||
if count == 0 {
|
||||
log.Println("data_sources表不存在，跳过约束清理")
|
||||
return nil
|
||||
}
|
||||
|
||||
// 检查是否有CHECK约束
|
||||
var constraintCount int64
|
||||
err = DB.Raw("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='data_sources' AND sql LIKE '%CHECK%'").Scan(&constraintCount).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 如果没有CHECK约束，直接返回
|
||||
if constraintCount == 0 {
|
||||
log.Println("data_sources表没有CHECK约束，跳过清理")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("检测到旧的CHECK约束，开始清理...")
|
||||
|
||||
// 重建表，去掉CHECK约束
|
||||
// 1. 创建新表（不包含CHECK约束）
|
||||
createNewTableSQL := `
|
||||
CREATE TABLE data_sources_new (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
endpoint_id INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
config TEXT NOT NULL,
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
last_sync DATETIME,
|
||||
created_at DATETIME,
|
||||
updated_at DATETIME,
|
||||
deleted_at DATETIME
|
||||
)`
|
||||
|
||||
if err := DB.Exec(createNewTableSQL).Error; err != nil {
|
||||
return fmt.Errorf("创建新表失败: %w", err)
|
||||
}
|
||||
|
||||
// 2. 复制数据
|
||||
copyDataSQL := `
|
||||
INSERT INTO data_sources_new (id, endpoint_id, name, type, config, is_active, last_sync, created_at, updated_at, deleted_at)
|
||||
SELECT id, endpoint_id, name, type, config, is_active, last_sync, created_at, updated_at, deleted_at
|
||||
FROM data_sources`
|
||||
|
||||
if err := DB.Exec(copyDataSQL).Error; err != nil {
|
||||
return fmt.Errorf("复制数据失败: %w", err)
|
||||
}
|
||||
|
||||
// 3. 删除旧表
|
||||
if err := DB.Exec("DROP TABLE data_sources").Error; err != nil {
|
||||
return fmt.Errorf("删除旧表失败: %w", err)
|
||||
}
|
||||
|
||||
// 4. 重命名新表
|
||||
if err := DB.Exec("ALTER TABLE data_sources_new RENAME TO data_sources").Error; err != nil {
|
||||
return fmt.Errorf("重命名表失败: %w", err)
|
||||
}
|
||||
|
||||
log.Println("旧的CHECK约束清理完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close 关闭数据库连接
|
||||
func Close() error {
|
||||
if DB != nil {
|
||||
|
||||
18
go.mod
18
go.mod
@ -5,12 +5,30 @@ go 1.23.0
|
||||
toolchain go1.23.1
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.4
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.16
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.69
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2
|
||||
github.com/glebarez/sqlite v1.11.0
|
||||
golang.org/x/time v0.12.0
|
||||
gorm.io/gorm v1.30.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 // indirect
|
||||
github.com/aws/smithy-go v1.22.2 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/glebarez/go-sqlite v1.21.2 // indirect
|
||||
github.com/google/uuid v1.5.0 // indirect
|
||||
|
||||
36
go.sum
36
go.sum
@ -1,3 +1,39 @@
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.4 h1:GySzjhVvx0ERP6eyfAbAuAXLtAda5TEy19E5q5W8I9E=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.4/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.16 h1:XkruGnXX1nEZ+Nyo9v84TzsX+nj86icbFAeust6uo8A=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.16/go.mod h1:uCW7PNjGwZ5cOGZ5jr8vCWrYkGIhPoTNV23Q/tpHKzg=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.69 h1:8B8ZQboRc3uaIKjshve/XlvJ570R7BKNy3gftSbS178=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.69/go.mod h1:gPME6I8grR1jCqBFEGthULiolzf/Sexq/Wy42ibKK9c=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 h1:oQWSGexYasNpYp4epLGZxxjsDo8BMBh6iNWkTXQvkwk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31/go.mod h1:nc332eGUU+djP3vrMI6blS0woaCfHTe3KiSQUVTMRq0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 h1:o1v1VFfPcDVlK3ll1L5xHsaQAFdNtZ5GXnNR7SwueC4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35/go.mod h1:rZUQNYMNG+8uZxz9FOerQJ+FceCiodXvixpeRtdESrU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 h1:R5b82ubO2NntENm3SAm0ADME+H630HomNJdgv+yZ3xw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35/go.mod h1:FuA+nmgMRfkzVKYDNEqQadvEMxtxl9+RLT9ribCwEMs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 h1:th/m+Q18CkajTw1iqx2cKkLCij/uz8NMwJFPK91p2ug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35/go.mod h1:dkJuf0a1Bc8HAA0Zm2MoTGm/WDC18Td9vSbrQ1+VqE8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 h1:VHPZakq2L7w+RLzV54LmQavbvheFaR2u1NomJRSEfcU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3/go.mod h1:DX1e/lkbsAt0MkY3NgLYuH4jQvRfw8MYxTe9feR7aXM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 h1:/ldKrPPXTC421bTNWrUIpq3CxwHwRI/kpc+jPUTJocM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16/go.mod h1:5vkf/Ws0/wgIMJDQbjI4p2op86hNW6Hie5QtebrDgT8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 h1:2HuI7vWKhFWsBhIr2Zq8KfFZT6xqaId2XXnXZjkbEuc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16/go.mod h1:BrwWnsfbFtFeRjdx0iM1ymvlqDX1Oz68JsQaibX/wG8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 h1:T6Wu+8E2LeTUqzqQ/Bh1EoFNj1u4jUyveMgmTlu9fDU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2/go.mod h1:chSY8zfqmS0OnhZoO/hpPx/BHfAIL80m77HwhRLYScY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 h1:EU58LP8ozQDVroOEyAfcq0cGc5R/FTZjVoYJ6tvby3w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.4/go.mod h1:CrtOgCcysxMvrCoHnvNAD7PHWclmoFG78Q2xLK0KKcs=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 h1:XB4z0hbQtpmBnb1FQYvKaCM7UsS6Y/u8jVBwIUGeCTk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2/go.mod h1:hwRpqkRxnQ58J9blRDrB4IanlXCpcKmsC83EhG77upg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 h1:nyLjs8sYJShFYj6aiyjCBI3EcLn1udWrQTjEF+SOXB0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.21/go.mod h1:EhdxtZ+g84MSGrSrHzZiUm9PYiZkrADNja15wtRJSJo=
|
||||
github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
|
||||
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
|
||||
|
||||
@ -32,19 +32,34 @@ func InitData() error {
|
||||
|
||||
// 4. 统计需要预加载的数据源
|
||||
var activeDataSources []model.DataSource
|
||||
var totalDataSources, disabledDataSources, apiDataSources int
|
||||
|
||||
for _, endpoint := range endpoints {
|
||||
if !endpoint.IsActive {
|
||||
continue
|
||||
}
|
||||
for _, ds := range endpoint.DataSources {
|
||||
if ds.IsActive && ds.Type != "api_get" && ds.Type != "api_post" {
|
||||
// API类型的数据源不需要预加载，使用实时请求
|
||||
activeDataSources = append(activeDataSources, ds)
|
||||
totalDataSources++
|
||||
|
||||
if !ds.IsActive {
|
||||
disabledDataSources++
|
||||
log.Printf("跳过禁用的数据源: %s (ID: %d)", ds.Name, ds.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
if ds.Type == "api_get" || ds.Type == "api_post" {
|
||||
apiDataSources++
|
||||
log.Printf("跳过API类型数据源: %s (ID: %d, 类型: %s) - 使用实时请求", ds.Name, ds.ID, ds.Type)
|
||||
continue
|
||||
}
|
||||
|
||||
// 需要预加载的数据源
|
||||
activeDataSources = append(activeDataSources, ds)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("发现 %d 个端点，%d 个需要预加载的数据源", len(endpoints), len(activeDataSources))
|
||||
log.Printf("发现 %d 个端点，总共 %d 个数据源", len(endpoints), totalDataSources)
|
||||
log.Printf("其中: 禁用 %d 个，API类型 %d 个，需要预加载 %d 个", disabledDataSources, apiDataSources, len(activeDataSources))
|
||||
|
||||
if len(activeDataSources) == 0 {
|
||||
log.Println("✓ 没有需要预加载的数据源")
|
||||
|
||||
@ -29,7 +29,7 @@ type DataSource struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
EndpointID uint `json:"endpoint_id" gorm:"not null;index"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
Type string `json:"type" gorm:"not null;check:type IN ('lankong', 'manual', 'api_get', 'api_post', 'endpoint')"`
|
||||
Type string `json:"type" gorm:"not null"`
|
||||
Config string `json:"config" gorm:"not null"`
|
||||
IsActive bool `json:"is_active" gorm:"default:true"`
|
||||
LastSync *time.Time `json:"last_sync,omitempty"`
|
||||
@ -80,6 +80,9 @@ type DataSourceConfig struct {
|
||||
|
||||
// 端点配置
|
||||
EndpointConfig *EndpointConfig `json:"endpoint_config,omitempty"`
|
||||
|
||||
// S3配置
|
||||
S3Config *S3Config `json:"s3_config,omitempty"`
|
||||
}
|
||||
|
||||
type LankongConfig struct {
|
||||
@ -103,3 +106,26 @@ type APIConfig struct {
|
||||
type EndpointConfig struct {
|
||||
EndpointIDs []uint `json:"endpoint_ids"` // 选中的端点ID列表
|
||||
}
|
||||
|
||||
// S3Config S3通用协议配置
|
||||
type S3Config struct {
|
||||
// 基础配置
|
||||
Endpoint string `json:"endpoint"` // S3端点地址
|
||||
BucketName string `json:"bucket_name"` // 存储桶名称
|
||||
Region string `json:"region"` // 地区
|
||||
AccessKeyID string `json:"access_key_id"` // 访问密钥ID
|
||||
SecretAccessKey string `json:"secret_access_key"` // 访问密钥
|
||||
|
||||
// 高级配置
|
||||
ListObjectsVersion string `json:"list_objects_version"` // 列出对象版本 (v1/v2)，默认v2
|
||||
UsePathStyle bool `json:"use_path_style"` // 是否使用path style
|
||||
RemoveBucket bool `json:"remove_bucket"` // 是否从路径中删除bucket名称
|
||||
|
||||
// 自定义域名配置
|
||||
CustomDomain string `json:"custom_domain"` // 自定义访问域名，支持路径
|
||||
|
||||
// 文件过滤配置
|
||||
FolderPath string `json:"folder_path"` // 提取的文件夹路径，例如: /img
|
||||
IncludeSubfolders bool `json:"include_subfolders"` // 是否提取所有子文件夹
|
||||
FileExtensions []string `json:"file_extensions"` // 提取的文件格式后缀，支持正则匹配
|
||||
}
|
||||
|
||||
104
readme.md
104
readme.md
@ -4,7 +4,7 @@
|
||||
|
||||
## 功能特性
|
||||
|
||||
- 🎯 支持多种数据源：兰空图床API、手动配置、通用API接口
|
||||
- 🎯 支持多种数据源：兰空图床API, s3协议对象存储, 手动配置, 通用API接口(GET/POST)
|
||||
- 🔐 OAuth2.0 管理后台登录（CZL Connect）
|
||||
- 💾 SQLite数据库存储
|
||||
- ⚡ 内存缓存机制
|
||||
@ -12,107 +12,9 @@
|
||||
- 📝 可配置首页内容
|
||||
- 🎨 现代化Web管理界面
|
||||
|
||||
## 环境变量配置
|
||||
## 部署and讨论
|
||||
|
||||
复制 `env.example` 为 `.env` 并配置以下环境变量：
|
||||
|
||||
```bash
|
||||
# 服务器配置
|
||||
PORT=:5003 # 服务端口
|
||||
READ_TIMEOUT=30s # 读取超时
|
||||
WRITE_TIMEOUT=30s # 写入超时
|
||||
MAX_HEADER_BYTES=1048576 # 最大请求头大小
|
||||
|
||||
# 数据存储目录
|
||||
DATA_DIR=./data # 数据存储目录
|
||||
|
||||
# OAuth2.0 配置 (必需)
|
||||
OAUTH_CLIENT_ID=your-oauth-client-id # CZL Connect 客户端ID
|
||||
OAUTH_CLIENT_SECRET=your-oauth-client-secret # CZL Connect 客户端密钥
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
|
||||
1. 克隆项目
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd random-api-go
|
||||
```
|
||||
|
||||
2. 配置环境变量
|
||||
```bash
|
||||
cp env.example .env
|
||||
# 编辑 .env 文件，填入正确的 OAuth 配置
|
||||
```
|
||||
|
||||
3. 运行服务
|
||||
```bash
|
||||
go run main.go
|
||||
```
|
||||
|
||||
4. 访问服务
|
||||
- 首页: http://localhost:5003
|
||||
- 管理后台: http://localhost:5003/admin
|
||||
|
||||
## OAuth2.0 配置
|
||||
|
||||
本项目使用 CZL Connect 作为 OAuth2.0 提供商：
|
||||
|
||||
- 授权端点: https://connect.czl.net/oauth2/authorize
|
||||
- 令牌端点: https://connect.czl.net/api/oauth2/token
|
||||
- 用户信息端点: https://connect.czl.net/api/oauth2/userinfo
|
||||
|
||||
请在 CZL Connect 中注册应用并获取 `client_id` 和 `client_secret`。
|
||||
|
||||
## API 端点
|
||||
|
||||
### 公开API
|
||||
- `GET /` - 首页
|
||||
- `GET /{endpoint}` - 随机API端点
|
||||
|
||||
### 管理API
|
||||
- `GET /admin/api/oauth-config` - 获取OAuth配置
|
||||
- `POST /admin/api/oauth-verify` - 验证OAuth授权码
|
||||
- `GET /admin/api/endpoints` - 列出所有端点
|
||||
- `POST /admin/api/endpoints/` - 创建端点
|
||||
- `GET /admin/api/endpoints/{id}` - 获取端点详情
|
||||
- `PUT /admin/api/endpoints/{id}` - 更新端点
|
||||
- `DELETE /admin/api/endpoints/{id}` - 删除端点
|
||||
- `POST /admin/api/data-sources` - 创建数据源
|
||||
- `GET /admin/api/url-replace-rules` - 列出URL替换规则
|
||||
- `POST /admin/api/url-replace-rules/` - 创建URL替换规则
|
||||
- `GET /admin/api/home-config` - 获取首页配置
|
||||
- `PUT /admin/api/home-config/` - 更新首页配置
|
||||
|
||||
## 数据源类型
|
||||
|
||||
1. **兰空图床 (lankong)**: 从兰空图床API获取图片
|
||||
2. **手动配置 (manual)**: 手动配置的URL列表
|
||||
3. **API GET (api_get)**: 从GET接口获取数据
|
||||
4. **API POST (api_post)**: 从POST接口获取数据
|
||||
|
||||
## 部署
|
||||
|
||||
### Docker 部署
|
||||
|
||||
```dockerfile
|
||||
FROM golang:1.21-alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN go build -o random-api-server main.go
|
||||
|
||||
FROM alpine:latest
|
||||
RUN apk --no-cache add ca-certificates
|
||||
WORKDIR /root/
|
||||
COPY --from=builder /app/random-api-server .
|
||||
COPY --from=builder /app/web ./web
|
||||
EXPOSE 5003
|
||||
CMD ["./random-api-server"]
|
||||
```
|
||||
|
||||
### 环境变量部署
|
||||
|
||||
确保在生产环境中正确设置所有必需的环境变量，特别是OAuth配置。
|
||||
<https://www.q58.club/t/topic/127>
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@ -16,6 +16,7 @@ type DataSourceFetcher struct {
|
||||
cacheManager *CacheManager
|
||||
lankongFetcher *LankongFetcher
|
||||
apiFetcher *APIFetcher
|
||||
s3Fetcher *S3Fetcher
|
||||
}
|
||||
|
||||
// NewDataSourceFetcher 创建数据源获取器
|
||||
@ -38,6 +39,7 @@ func NewDataSourceFetcher(cacheManager *CacheManager) *DataSourceFetcher {
|
||||
cacheManager: cacheManager,
|
||||
lankongFetcher: lankongFetcher,
|
||||
apiFetcher: NewAPIFetcher(),
|
||||
s3Fetcher: NewS3Fetcher(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -86,6 +88,8 @@ func (dsf *DataSourceFetcher) FetchURLs(dataSource *model.DataSource) ([]string,
|
||||
urls, err = dsf.fetchManualURLs(dataSource)
|
||||
case "endpoint":
|
||||
urls, err = dsf.fetchEndpointURLs(dataSource)
|
||||
case "s3":
|
||||
urls, err = dsf.fetchS3URLs(dataSource)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported data source type: %s", dataSource.Type)
|
||||
}
|
||||
@ -180,6 +184,16 @@ func (dsf *DataSourceFetcher) fetchEndpointURLs(dataSource *model.DataSource) ([
|
||||
return urls, nil
|
||||
}
|
||||
|
||||
// fetchS3URLs 获取S3存储桶URL
|
||||
func (dsf *DataSourceFetcher) fetchS3URLs(dataSource *model.DataSource) ([]string, error) {
|
||||
var config model.S3Config
|
||||
if err := json.Unmarshal([]byte(dataSource.Config), &config); err != nil {
|
||||
return nil, fmt.Errorf("invalid S3 config: %w", err)
|
||||
}
|
||||
|
||||
return dsf.s3Fetcher.FetchURLs(&config)
|
||||
}
|
||||
|
||||
// updateDataSourceSyncTime 更新数据源的同步时间
|
||||
func (dsf *DataSourceFetcher) updateDataSourceSyncTime(dataSource *model.DataSource) error {
|
||||
// 这里需要导入database包来更新数据库
|
||||
|
||||
@ -22,6 +22,26 @@ type EndpointService struct {
|
||||
var endpointService *EndpointService
|
||||
var once sync.Once
|
||||
|
||||
// 支持的数据源类型列表
|
||||
var supportedDataSourceTypes = []string{
|
||||
"lankong",
|
||||
"manual",
|
||||
"api_get",
|
||||
"api_post",
|
||||
"endpoint",
|
||||
"s3",
|
||||
}
|
||||
|
||||
// validateDataSourceType 验证数据源类型
|
||||
func validateDataSourceType(dataSourceType string) error {
|
||||
for _, supportedType := range supportedDataSourceTypes {
|
||||
if dataSourceType == supportedType {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("unsupported data source type: %s, supported types: %v", dataSourceType, supportedDataSourceTypes)
|
||||
}
|
||||
|
||||
// GetEndpointService 获取端点服务单例
|
||||
func GetEndpointService() *EndpointService {
|
||||
once.Do(func() {
|
||||
@ -287,6 +307,11 @@ func (s *EndpointService) applyURLReplaceRules(url, endpointURL string) string {
|
||||
|
||||
// CreateDataSource 创建数据源
|
||||
func (s *EndpointService) CreateDataSource(dataSource *model.DataSource) error {
|
||||
// 验证数据源类型
|
||||
if err := validateDataSourceType(dataSource.Type); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.DB.Create(dataSource).Error; err != nil {
|
||||
return fmt.Errorf("failed to create data source: %w", err)
|
||||
}
|
||||
@ -304,6 +329,11 @@ func (s *EndpointService) CreateDataSource(dataSource *model.DataSource) error {
|
||||
|
||||
// UpdateDataSource 更新数据源
|
||||
func (s *EndpointService) UpdateDataSource(dataSource *model.DataSource) error {
|
||||
// 验证数据源类型
|
||||
if err := validateDataSourceType(dataSource.Type); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.DB.Save(dataSource).Error; err != nil {
|
||||
return fmt.Errorf("failed to update data source: %w", err)
|
||||
}
|
||||
|
||||
@ -74,6 +74,12 @@ func (p *Preloader) ResumePeriodicRefresh() {
|
||||
|
||||
// PreloadDataSourceOnSave 在保存数据源时预加载数据
|
||||
func (p *Preloader) PreloadDataSourceOnSave(dataSource *model.DataSource) {
|
||||
// 检查数据源是否处于活跃状态
|
||||
if !dataSource.IsActive {
|
||||
log.Printf("数据源 %d 已禁用，跳过预加载", dataSource.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// API类型的数据源不需要预加载，使用实时请求
|
||||
if dataSource.Type == "api_get" || dataSource.Type == "api_post" {
|
||||
log.Printf("API数据源 %d (%s) 使用实时请求，跳过预加载", dataSource.ID, dataSource.Type)
|
||||
@ -135,6 +141,18 @@ func (p *Preloader) RefreshDataSource(dataSourceID uint) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// 检查数据源是否处于活跃状态
|
||||
if !dataSource.IsActive {
|
||||
log.Printf("数据源 %d 已禁用，跳过刷新", dataSourceID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// API类型的数据源不需要预加载，使用实时请求
|
||||
if dataSource.Type == "api_get" || dataSource.Type == "api_post" {
|
||||
log.Printf("API数据源 %d (%s) 使用实时请求，跳过刷新", dataSource.ID, dataSource.Type)
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Printf("手动刷新数据源 %d", dataSourceID)
|
||||
return p.dataSourceFetcher.PreloadDataSource(&dataSource)
|
||||
}
|
||||
@ -281,6 +299,8 @@ func (p *Preloader) shouldPeriodicRefresh(dataSource *model.DataSource) bool {
|
||||
switch dataSource.Type {
|
||||
case "lankong":
|
||||
refreshInterval = 24 * time.Hour // 兰空图床每24小时刷新一次
|
||||
case "s3":
|
||||
refreshInterval = 6 * time.Hour // S3存储每6小时刷新一次
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
330
service/s3_fetcher.go
Normal file
330
service/s3_fetcher.go
Normal file
@ -0,0 +1,330 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"random-api-go/model"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
// S3Fetcher S3获取器
|
||||
type S3Fetcher struct {
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// NewS3Fetcher 创建S3获取器
|
||||
func NewS3Fetcher() *S3Fetcher {
|
||||
return &S3Fetcher{
|
||||
timeout: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// FetchURLs 从S3存储桶获取文件URL列表
|
||||
func (sf *S3Fetcher) FetchURLs(s3Config *model.S3Config) ([]string, error) {
|
||||
if s3Config == nil {
|
||||
return nil, fmt.Errorf("S3配置不能为空")
|
||||
}
|
||||
|
||||
// 验证必需的配置
|
||||
if s3Config.Endpoint == "" {
|
||||
return nil, fmt.Errorf("S3端点地址不能为空")
|
||||
}
|
||||
if s3Config.BucketName == "" {
|
||||
return nil, fmt.Errorf("存储桶名称不能为空")
|
||||
}
|
||||
if s3Config.AccessKeyID == "" {
|
||||
return nil, fmt.Errorf("访问密钥ID不能为空")
|
||||
}
|
||||
if s3Config.SecretAccessKey == "" {
|
||||
return nil, fmt.Errorf("访问密钥不能为空")
|
||||
}
|
||||
|
||||
// 创建S3客户端
|
||||
client, err := sf.createS3Client(s3Config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建S3客户端失败: %w", err)
|
||||
}
|
||||
|
||||
// 获取对象列表
|
||||
objects, err := sf.listObjects(client, s3Config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取对象列表失败: %w", err)
|
||||
}
|
||||
|
||||
// 过滤和转换为URL
|
||||
urls := sf.convertObjectsToURLs(objects, s3Config)
|
||||
|
||||
log.Printf("从S3存储桶 %s 获取到 %d 个文件URL", s3Config.BucketName, len(urls))
|
||||
return urls, nil
|
||||
}
|
||||
|
||||
// createS3Client 创建S3客户端
|
||||
func (sf *S3Fetcher) createS3Client(s3Config *model.S3Config) (*s3.Client, error) {
|
||||
// 设置默认地区
|
||||
region := s3Config.Region
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
|
||||
// 创建凭证
|
||||
creds := credentials.NewStaticCredentialsProvider(
|
||||
s3Config.AccessKeyID,
|
||||
s3Config.SecretAccessKey,
|
||||
"",
|
||||
)
|
||||
|
||||
// 创建配置
|
||||
cfg, err := config.LoadDefaultConfig(context.TODO(),
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(creds),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("加载AWS配置失败: %w", err)
|
||||
}
|
||||
|
||||
// 创建S3客户端选项
|
||||
options := func(o *s3.Options) {
|
||||
if s3Config.Endpoint != "" {
|
||||
o.BaseEndpoint = aws.String(s3Config.Endpoint)
|
||||
}
|
||||
o.UsePathStyle = s3Config.UsePathStyle
|
||||
}
|
||||
|
||||
client := s3.NewFromConfig(cfg, options)
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// listObjects 列出存储桶中的对象
|
||||
func (sf *S3Fetcher) listObjects(client *s3.Client, s3Config *model.S3Config) ([]types.Object, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), sf.timeout)
|
||||
defer cancel()
|
||||
|
||||
var allObjects []types.Object
|
||||
var continuationToken *string
|
||||
|
||||
// 设置前缀（文件夹路径）
|
||||
prefix := strings.TrimPrefix(s3Config.FolderPath, "/")
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
// 设置分隔符（如果不包含子文件夹）
|
||||
var delimiter *string
|
||||
if !s3Config.IncludeSubfolders {
|
||||
delimiter = aws.String("/")
|
||||
}
|
||||
|
||||
// 确定使用的ListObjects版本
|
||||
listVersion := s3Config.ListObjectsVersion
|
||||
if listVersion == "" {
|
||||
listVersion = "v2" // 默认使用v2
|
||||
}
|
||||
|
||||
for {
|
||||
if listVersion == "v1" {
|
||||
// 使用ListObjects (v1)
|
||||
input := &s3.ListObjectsInput{
|
||||
Bucket: aws.String(s3Config.BucketName),
|
||||
Prefix: aws.String(prefix),
|
||||
Delimiter: delimiter,
|
||||
MaxKeys: aws.Int32(1000),
|
||||
}
|
||||
|
||||
if continuationToken != nil {
|
||||
input.Marker = continuationToken
|
||||
}
|
||||
|
||||
result, err := client.ListObjects(ctx, input)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ListObjects失败: %w", err)
|
||||
}
|
||||
|
||||
allObjects = append(allObjects, result.Contents...)
|
||||
|
||||
if !aws.ToBool(result.IsTruncated) {
|
||||
break
|
||||
}
|
||||
|
||||
if len(result.Contents) > 0 {
|
||||
continuationToken = result.Contents[len(result.Contents)-1].Key
|
||||
}
|
||||
} else {
|
||||
// 使用ListObjectsV2 (v2)
|
||||
input := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s3Config.BucketName),
|
||||
Prefix: aws.String(prefix),
|
||||
Delimiter: delimiter,
|
||||
MaxKeys: aws.Int32(1000),
|
||||
ContinuationToken: continuationToken,
|
||||
}
|
||||
|
||||
result, err := client.ListObjectsV2(ctx, input)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ListObjectsV2失败: %w", err)
|
||||
}
|
||||
|
||||
allObjects = append(allObjects, result.Contents...)
|
||||
|
||||
if !aws.ToBool(result.IsTruncated) {
|
||||
break
|
||||
}
|
||||
|
||||
continuationToken = result.NextContinuationToken
|
||||
}
|
||||
}
|
||||
|
||||
return allObjects, nil
|
||||
}
|
||||
|
||||
// convertObjectsToURLs 将S3对象转换为URL列表
|
||||
func (sf *S3Fetcher) convertObjectsToURLs(objects []types.Object, s3Config *model.S3Config) []string {
|
||||
var urls []string
|
||||
|
||||
// 编译文件扩展名正则表达式
|
||||
var extensionRegexes []*regexp.Regexp
|
||||
for _, ext := range s3Config.FileExtensions {
|
||||
if ext != "" {
|
||||
// 确保扩展名以点开头
|
||||
if !strings.HasPrefix(ext, ".") {
|
||||
ext = "." + ext
|
||||
}
|
||||
// 转义特殊字符并创建正则表达式
|
||||
pattern := regexp.QuoteMeta(ext) + "$"
|
||||
if regex, err := regexp.Compile("(?i)" + pattern); err == nil {
|
||||
extensionRegexes = append(extensionRegexes, regex)
|
||||
} else {
|
||||
log.Printf("警告: 无效的文件扩展名正则表达式 '%s': %v", ext, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, obj := range objects {
|
||||
if obj.Key == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
key := aws.ToString(obj.Key)
|
||||
|
||||
// 跳过以/结尾的对象（文件夹）
|
||||
if strings.HasSuffix(key, "/") {
|
||||
continue
|
||||
}
|
||||
|
||||
// 如果设置了文件扩展名过滤，检查是否匹配
|
||||
if len(extensionRegexes) > 0 {
|
||||
matched := false
|
||||
for _, regex := range extensionRegexes {
|
||||
if regex.MatchString(key) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 生成URL
|
||||
fileURL := sf.generateURL(key, s3Config)
|
||||
if fileURL != "" {
|
||||
urls = append(urls, fileURL)
|
||||
}
|
||||
}
|
||||
|
||||
return urls
|
||||
}
|
||||
|
||||
// generateURL 生成文件的访问URL
|
||||
func (sf *S3Fetcher) generateURL(key string, s3Config *model.S3Config) string {
|
||||
// 如果设置了自定义域名
|
||||
if s3Config.CustomDomain != "" {
|
||||
return sf.generateCustomDomainURL(key, s3Config)
|
||||
}
|
||||
|
||||
// 使用S3标准URL格式
|
||||
return sf.generateS3URL(key, s3Config)
|
||||
}
|
||||
|
||||
// generateCustomDomainURL 生成自定义域名URL
|
||||
func (sf *S3Fetcher) generateCustomDomainURL(key string, s3Config *model.S3Config) string {
|
||||
baseURL := strings.TrimSuffix(s3Config.CustomDomain, "/")
|
||||
|
||||
// 处理key路径
|
||||
path := key
|
||||
if s3Config.RemoveBucket {
|
||||
// 如果需要移除bucket名称，并且key以bucket名称开头
|
||||
bucketPrefix := s3Config.BucketName + "/"
|
||||
if strings.HasPrefix(path, bucketPrefix) {
|
||||
path = strings.TrimPrefix(path, bucketPrefix)
|
||||
}
|
||||
}
|
||||
|
||||
// 对路径进行适当的URL编码，但保留路径分隔符
|
||||
path = sf.encodeURLPath(path)
|
||||
|
||||
// 确保路径以/开头
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
path = "/" + path
|
||||
}
|
||||
|
||||
return baseURL + path
|
||||
}
|
||||
|
||||
// generateS3URL 生成S3标准URL
|
||||
func (sf *S3Fetcher) generateS3URL(key string, s3Config *model.S3Config) string {
|
||||
// 对key进行适当的URL编码，但保留路径分隔符
|
||||
encodedKey := sf.encodeURLPath(key)
|
||||
|
||||
if s3Config.UsePathStyle {
|
||||
// Path-style URL: http://endpoint/bucket/key
|
||||
endpoint := strings.TrimSuffix(s3Config.Endpoint, "/")
|
||||
return fmt.Sprintf("%s/%s/%s", endpoint, s3Config.BucketName, encodedKey)
|
||||
} else {
|
||||
// Virtual-hosted-style URL: http://bucket.endpoint/key
|
||||
endpoint := strings.TrimSuffix(s3Config.Endpoint, "/")
|
||||
|
||||
// 解析endpoint以获取主机名
|
||||
if parsedURL, err := url.Parse(endpoint); err == nil {
|
||||
scheme := parsedURL.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "https"
|
||||
}
|
||||
host := parsedURL.Host
|
||||
if host == "" {
|
||||
host = parsedURL.Path
|
||||
}
|
||||
return fmt.Sprintf("%s://%s.%s/%s", scheme, s3Config.BucketName, host, encodedKey)
|
||||
}
|
||||
|
||||
// 如果解析失败，回退到path-style
|
||||
return fmt.Sprintf("%s/%s/%s", endpoint, s3Config.BucketName, encodedKey)
|
||||
}
|
||||
}
|
||||
|
||||
// encodeURLPath 对URL路径进行编码，保留路径分隔符，但编码其他特殊字符
|
||||
func (sf *S3Fetcher) encodeURLPath(path string) string {
|
||||
// 分割路径为各个部分
|
||||
parts := strings.Split(path, "/")
|
||||
|
||||
// 对每个部分进行URL编码
|
||||
for i, part := range parts {
|
||||
if part != "" {
|
||||
// 使用PathEscape对每个路径段进行编码
|
||||
// 这会将空格编码为%20，这是URL路径中的标准做法
|
||||
parts[i] = url.PathEscape(part)
|
||||
}
|
||||
}
|
||||
|
||||
// 重新组合路径
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
@ -11,7 +11,7 @@ import { Trash2, Plus } from 'lucide-react'
|
||||
import { authenticatedFetch } from '@/lib/auth'
|
||||
|
||||
interface DataSourceConfigFormProps {
|
||||
type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint'
|
||||
type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' | 's3'
|
||||
config: string
|
||||
onChange: (config: string) => void
|
||||
}
|
||||
@ -40,6 +40,21 @@ interface EndpointConfig {
|
||||
endpoint_ids: number[]
|
||||
}
|
||||
|
||||
interface S3Config {
|
||||
endpoint: string
|
||||
bucket_name: string
|
||||
region: string
|
||||
access_key_id: string
|
||||
secret_access_key: string
|
||||
list_objects_version: string
|
||||
use_path_style: boolean
|
||||
remove_bucket: boolean
|
||||
custom_domain: string
|
||||
folder_path: string
|
||||
include_subfolders: boolean
|
||||
file_extensions: string[]
|
||||
}
|
||||
|
||||
export default function DataSourceConfigForm({ type, config, onChange }: DataSourceConfigFormProps) {
|
||||
const [lankongConfig, setLankongConfig] = useState<LankongConfig>({
|
||||
api_token: '',
|
||||
@ -59,9 +74,25 @@ export default function DataSourceConfigForm({ type, config, onChange }: DataSou
|
||||
endpoint_ids: []
|
||||
})
|
||||
|
||||
const [s3Config, setS3Config] = useState<S3Config>({
|
||||
endpoint: '',
|
||||
bucket_name: '',
|
||||
region: '',
|
||||
access_key_id: '',
|
||||
secret_access_key: '',
|
||||
list_objects_version: 'v2',
|
||||
use_path_style: false,
|
||||
remove_bucket: false,
|
||||
custom_domain: '',
|
||||
folder_path: '',
|
||||
include_subfolders: true,
|
||||
file_extensions: []
|
||||
})
|
||||
|
||||
const [availableEndpoints, setAvailableEndpoints] = useState<Array<{id: number, name: string, url: string}>>([])
|
||||
|
||||
const [headerPairs, setHeaderPairs] = useState<Array<{key: string, value: string}>>([{key: '', value: ''}])
|
||||
const [extensionInputs, setExtensionInputs] = useState<string[]>([''])
|
||||
const [savedTokens, setSavedTokens] = useState<SavedToken[]>([])
|
||||
|
||||
const [newTokenName, setNewTokenName] = useState<string>('')
|
||||
@ -127,6 +158,26 @@ export default function DataSourceConfigForm({ type, config, onChange }: DataSou
|
||||
setEndpointConfig({
|
||||
endpoint_ids: parsed.endpoint_ids || []
|
||||
})
|
||||
} else if (type === 's3') {
|
||||
setS3Config({
|
||||
endpoint: parsed.endpoint || '',
|
||||
bucket_name: parsed.bucket_name || '',
|
||||
region: parsed.region || '',
|
||||
access_key_id: parsed.access_key_id || '',
|
||||
secret_access_key: parsed.secret_access_key || '',
|
||||
list_objects_version: parsed.list_objects_version || 'v2',
|
||||
use_path_style: parsed.use_path_style || false,
|
||||
remove_bucket: parsed.remove_bucket || false,
|
||||
custom_domain: parsed.custom_domain || '',
|
||||
folder_path: parsed.folder_path || '',
|
||||
include_subfolders: parsed.include_subfolders !== false,
|
||||
file_extensions: parsed.file_extensions || []
|
||||
})
|
||||
|
||||
// 设置文件扩展名输入框
|
||||
const extensions = parsed.file_extensions || ['']
|
||||
if (extensions.length === 0) extensions.push('')
|
||||
setExtensionInputs(extensions)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to parse config:', error)
|
||||
@ -257,6 +308,33 @@ export default function DataSourceConfigForm({ type, config, onChange }: DataSou
|
||||
updateEndpointConfig(newIds)
|
||||
}
|
||||
|
||||
// 更新S3配置
|
||||
const updateS3Config = (field: keyof S3Config, value: string | boolean | string[]) => {
|
||||
const newConfig = { ...s3Config, [field]: value }
|
||||
setS3Config(newConfig)
|
||||
onChange(JSON.stringify(newConfig))
|
||||
}
|
||||
|
||||
// 添加文件扩展名
|
||||
const addExtension = () => {
|
||||
const newExtensions = [...extensionInputs, '']
|
||||
setExtensionInputs(newExtensions)
|
||||
}
|
||||
|
||||
// 删除文件扩展名
|
||||
const removeExtension = (index: number) => {
|
||||
const newExtensions = extensionInputs.filter((_, i) => i !== index)
|
||||
setExtensionInputs(newExtensions)
|
||||
updateS3Config('file_extensions', newExtensions.filter(ext => ext.trim() !== ''))
|
||||
}
|
||||
|
||||
// 更新文件扩展名
|
||||
const updateExtension = (index: number, value: string) => {
|
||||
const newExtensions = extensionInputs.map((ext, i) => i === index ? value : ext)
|
||||
setExtensionInputs(newExtensions)
|
||||
updateS3Config('file_extensions', newExtensions.filter(ext => ext.trim() !== ''))
|
||||
}
|
||||
|
||||
if (type === 'manual') {
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
@ -560,5 +638,193 @@ export default function DataSourceConfigForm({ type, config, onChange }: DataSou
|
||||
)
|
||||
}
|
||||
|
||||
if (type === 's3') {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{/* 基础配置 */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="text-sm">基础配置</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="s3-endpoint">S3端点地址</Label>
|
||||
<Input
|
||||
id="s3-endpoint"
|
||||
value={s3Config.endpoint}
|
||||
onChange={(e) => updateS3Config('endpoint', e.target.value)}
|
||||
placeholder="https://s3.amazonaws.com"
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="s3-bucket">存储桶名称</Label>
|
||||
<Input
|
||||
id="s3-bucket"
|
||||
value={s3Config.bucket_name}
|
||||
onChange={(e) => updateS3Config('bucket_name', e.target.value)}
|
||||
placeholder="my-bucket"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="s3-region">地区</Label>
|
||||
<Input
|
||||
id="s3-region"
|
||||
value={s3Config.region}
|
||||
onChange={(e) => updateS3Config('region', e.target.value)}
|
||||
placeholder="us-east-1"
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="s3-version">列出对象版本</Label>
|
||||
<select
|
||||
id="s3-version"
|
||||
value={s3Config.list_objects_version}
|
||||
onChange={(e) => updateS3Config('list_objects_version', e.target.value)}
|
||||
className="flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
<option value="v2">v2</option>
|
||||
<option value="v1">v1</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="s3-access-key">访问密钥ID</Label>
|
||||
<Input
|
||||
id="s3-access-key"
|
||||
value={s3Config.access_key_id}
|
||||
onChange={(e) => updateS3Config('access_key_id', e.target.value)}
|
||||
placeholder="AKIAIOSFODNN7EXAMPLE"
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="s3-secret-key">访问密钥</Label>
|
||||
<Input
|
||||
id="s3-secret-key"
|
||||
type="password"
|
||||
value={s3Config.secret_access_key}
|
||||
onChange={(e) => updateS3Config('secret_access_key', e.target.value)}
|
||||
placeholder="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* 高级配置 */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="text-sm">高级配置</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
<div className="flex items-center space-x-2">
|
||||
<Checkbox
|
||||
id="s3-path-style"
|
||||
checked={s3Config.use_path_style}
|
||||
onCheckedChange={(checked) => updateS3Config('use_path_style', checked as boolean)}
|
||||
/>
|
||||
<Label htmlFor="s3-path-style">使用Path Style URL</Label>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center space-x-2">
|
||||
<Checkbox
|
||||
id="s3-remove-bucket"
|
||||
checked={s3Config.remove_bucket}
|
||||
onCheckedChange={(checked) => updateS3Config('remove_bucket', checked as boolean)}
|
||||
/>
|
||||
<Label htmlFor="s3-remove-bucket">从路径中删除bucket名称</Label>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="s3-custom-domain">自定义访问域名（可选）</Label>
|
||||
<Input
|
||||
id="s3-custom-domain"
|
||||
value={s3Config.custom_domain}
|
||||
onChange={(e) => updateS3Config('custom_domain', e.target.value)}
|
||||
placeholder="https://cdn.example.com"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
留空使用S3标准URL，支持路径如: https://cdn.example.com/path
|
||||
</p>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* 文件过滤配置 */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="text-sm">文件过滤配置</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="s3-folder-path">文件夹路径（可选）</Label>
|
||||
<Input
|
||||
id="s3-folder-path"
|
||||
value={s3Config.folder_path}
|
||||
onChange={(e) => updateS3Config('folder_path', e.target.value)}
|
||||
placeholder="/images"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
指定要提取的文件夹路径，如: /images 或 /uploads/photos
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center space-x-2">
|
||||
<Checkbox
|
||||
id="s3-include-subfolders"
|
||||
checked={s3Config.include_subfolders}
|
||||
onCheckedChange={(checked) => updateS3Config('include_subfolders', checked as boolean)}
|
||||
/>
|
||||
<Label htmlFor="s3-include-subfolders">包含所有子文件夹</Label>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label>文件格式过滤</Label>
|
||||
{extensionInputs.map((ext, index) => (
|
||||
<div key={index} className="flex gap-2">
|
||||
<Input
|
||||
value={ext}
|
||||
onChange={(e) => updateExtension(index, e.target.value)}
|
||||
placeholder=".jpg 或 .png"
|
||||
className="flex-1"
|
||||
/>
|
||||
{extensionInputs.length > 1 && (
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => removeExtension(index)}
|
||||
className="text-red-600 hover:text-red-700"
|
||||
>
|
||||
<Trash2 className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={addExtension}
|
||||
className="w-full"
|
||||
>
|
||||
<Plus className="h-4 w-4 mr-1" />
|
||||
添加文件格式
|
||||
</Button>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
留空表示不过滤文件格式，支持正则匹配如: .jpg, .png, .gif
|
||||
</p>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
@ -26,7 +26,7 @@ export default function DataSourceManagement({
|
||||
const [editingDataSource, setEditingDataSource] = useState<DataSource | null>(null)
|
||||
const [formData, setFormData] = useState({
|
||||
name: '',
|
||||
type: 'manual' as 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint',
|
||||
type: 'manual' as 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' | 's3',
|
||||
config: '',
|
||||
is_active: true
|
||||
})
|
||||
@ -185,6 +185,7 @@ export default function DataSourceManagement({
|
||||
case 'api_get': return 'GET接口'
|
||||
case 'api_post': return 'POST接口'
|
||||
case 'endpoint': return '已有端点'
|
||||
case 's3': return 'S3对象存储'
|
||||
default: return type
|
||||
}
|
||||
}
|
||||
@ -245,7 +246,7 @@ export default function DataSourceManagement({
|
||||
<select
|
||||
id="ds-type"
|
||||
value={formData.type}
|
||||
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' })}
|
||||
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' | 's3' })}
|
||||
className="flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
<option value="manual">手动数据链接</option>
|
||||
@ -253,6 +254,7 @@ export default function DataSourceManagement({
|
||||
<option value="api_get">GET接口</option>
|
||||
<option value="api_post">POST接口</option>
|
||||
<option value="endpoint">已有端点</option>
|
||||
<option value="s3">S3对象存储</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -18,6 +18,14 @@ const nextConfig: NextConfig = {
|
||||
{
|
||||
source: "/api/:path*",
|
||||
destination: "http://localhost:5003/api/:path*",
|
||||
},
|
||||
{
|
||||
source: "/pic/:path*",
|
||||
destination: "http://localhost:5003/pic/:path*",
|
||||
},
|
||||
{
|
||||
source: "/video/:path*",
|
||||
destination: "http://localhost:5003/video/:path*",
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
@ -21,7 +21,7 @@ export interface DataSource {
|
||||
id: number
|
||||
endpoint_id: number
|
||||
name: string
|
||||
type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint'
|
||||
type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' | 's3'
|
||||
config: string
|
||||
is_active: boolean
|
||||
last_sync?: string
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user