feat(config): Add max file size threshold and improve path configuration

- Extend PathConfig with MaxSize parameter to define upper file size limit
- Update routing logic to handle both minimum and maximum file size thresholds
- Enhance frontend configuration UI to support max size input
- Improve file routing decision-making with comprehensive size range checks
This commit is contained in:
wood chen 2025-02-17 18:12:16 +08:00
parent 69050adf57
commit f7a52a1be5
3 changed files with 239 additions and 129 deletions

View File

@ -12,9 +12,11 @@ type Config struct {
} }
type PathConfig struct { type PathConfig struct {
DefaultTarget string `json:"DefaultTarget"` // 默认回源地址 Path string `json:"Path"`
ExtensionMap map[string]string `json:"ExtensionMap"` // 特定后缀的回源地址 DefaultTarget string `json:"DefaultTarget"`
SizeThreshold int64 `json:"SizeThreshold"` // 文件大小阈值(字节)超过此大小才使用ExtensionMap ExtensionMap map[string]string `json:"ExtensionMap"`
SizeThreshold int64 `json:"SizeThreshold"` // 最小文件大小阈值
MaxSize int64 `json:"MaxSize"` // 最大文件大小阈值
processedExtMap map[string]string // 内部使用,存储拆分后的映射 processedExtMap map[string]string // 内部使用,存储拆分后的映射
} }

View File

@ -22,10 +22,19 @@ type fileSizeCache struct {
timestamp time.Time timestamp time.Time
} }
// 可访问性缓存项
type accessibilityCache struct {
accessible bool
timestamp time.Time
}
var ( var (
// 文件大小缓存过期时间5分钟 // 文件大小缓存过期时间5分钟
sizeCache sync.Map sizeCache sync.Map
// 可访问性缓存过期时间30秒
accessCache sync.Map
cacheTTL = 5 * time.Minute cacheTTL = 5 * time.Minute
accessTTL = 30 * time.Second
maxCacheSize = 10000 // 最大缓存条目数 maxCacheSize = 10000 // 最大缓存条目数
) )
@ -35,6 +44,7 @@ func init() {
ticker := time.NewTicker(time.Minute) ticker := time.NewTicker(time.Minute)
for range ticker.C { for range ticker.C {
now := time.Now() now := time.Now()
// 清理文件大小缓存
var items []struct { var items []struct {
key interface{} key interface{}
timestamp time.Time timestamp time.Time
@ -59,6 +69,15 @@ func init() {
sizeCache.Delete(items[i].key) sizeCache.Delete(items[i].key)
} }
} }
// 清理可访问性缓存
accessCache.Range(func(key, value interface{}) bool {
cache := value.(accessibilityCache)
if now.Sub(cache.timestamp) > accessTTL {
accessCache.Delete(key)
}
return true
})
} }
}() }()
} }
@ -168,10 +187,16 @@ func GetTargetURL(client *http.Client, r *http.Request, pathConfig config.PathCo
// 默认使用默认目标 // 默认使用默认目标
targetBase := pathConfig.DefaultTarget targetBase := pathConfig.DefaultTarget
// 如果没有设置阈值,使用默认值 500KB // 如果没有设置最小阈值,使用默认值 500KB
threshold := pathConfig.SizeThreshold minThreshold := pathConfig.SizeThreshold
if threshold <= 0 { if minThreshold <= 0 {
threshold = 500 * 1024 minThreshold = 500 * 1024
}
// 如果没有设置最大阈值,使用默认值 10MB
maxThreshold := pathConfig.MaxSize
if maxThreshold <= 0 {
maxThreshold = 10 * 1024 * 1024
} }
// 检查文件扩展名 // 检查文件扩展名
@ -181,45 +206,59 @@ func GetTargetURL(client *http.Client, r *http.Request, pathConfig config.PathCo
ext = ext[1:] // 移除开头的点 ext = ext[1:] // 移除开头的点
// 先检查是否在扩展名映射中 // 先检查是否在扩展名映射中
if altTarget, exists := pathConfig.GetExtensionTarget(ext); exists { if altTarget, exists := pathConfig.GetExtensionTarget(ext); exists {
// 先检查扩展名映射的目标是否可访问 // 使用 channel 来并发获取文件大小和检查可访问性
if isTargetAccessible(client, altTarget+path) { type result struct {
// 检查文件大小 size int64
contentLength := r.ContentLength accessible bool
if contentLength <= 0 { err error
// 如果无法获取 Content-Length尝试发送 HEAD 请求到备用源 }
if size, err := GetFileSize(client, altTarget+path); err == nil { defaultChan := make(chan result, 1)
contentLength = size altChan := make(chan result, 1)
log.Printf("[FileSize] Path: %s, Size: %s (from alternative source)",
path, FormatBytes(contentLength)) // 并发检查默认源和备用源
} else { go func() {
log.Printf("[FileSize] Failed to get size from alternative source for %s: %v", path, err) size, err := GetFileSize(client, targetBase+path)
// 如果从备用源获取失败,尝试从默认源获取 defaultChan <- result{size: size, err: err}
if size, err := GetFileSize(client, targetBase+path); err == nil { }()
contentLength = size go func() {
accessible := isTargetAccessible(client, altTarget+path)
altChan <- result{accessible: accessible}
}()
// 获取默认源结果
defaultResult := <-defaultChan
if defaultResult.err != nil {
log.Printf("[FileSize] Failed to get size from default source for %s: %v", path, defaultResult.err)
return targetBase
}
contentLength := defaultResult.size
log.Printf("[FileSize] Path: %s, Size: %s (from default source)", log.Printf("[FileSize] Path: %s, Size: %s (from default source)",
path, FormatBytes(contentLength)) path, FormatBytes(contentLength))
} else {
log.Printf("[FileSize] Failed to get size from default source for %s: %v", path, err)
}
}
} else {
log.Printf("[FileSize] Path: %s, Size: %s (from Content-Length)",
path, FormatBytes(contentLength))
}
// 只有当文件大于阈值时才使用扩展名映射的目标 // 检查文件大小是否在阈值范围内
if contentLength > threshold { if contentLength > minThreshold && contentLength <= maxThreshold {
log.Printf("[Route] %s -> %s (size: %s > %s)", // 获取备用源检查结果
path, altTarget, FormatBytes(contentLength), FormatBytes(threshold)) altResult := <-altChan
targetBase = altTarget if altResult.accessible {
} else { log.Printf("[Route] %s -> %s (size: %s > %s and <= %s)",
log.Printf("[Route] %s -> %s (size: %s <= %s)", path, altTarget, FormatBytes(contentLength),
path, targetBase, FormatBytes(contentLength), FormatBytes(threshold)) FormatBytes(minThreshold), FormatBytes(maxThreshold))
} return altTarget
} else { } else {
log.Printf("[Route] %s -> %s (fallback: alternative target not accessible)", log.Printf("[Route] %s -> %s (fallback: alternative target not accessible)",
path, targetBase) path, targetBase)
} }
} else if contentLength <= minThreshold {
// 如果文件大小不合适,直接丢弃备用源检查结果
go func() { <-altChan }()
log.Printf("[Route] %s -> %s (size: %s <= %s)",
path, targetBase, FormatBytes(contentLength), FormatBytes(minThreshold))
} else {
// 如果文件大小不合适,直接丢弃备用源检查结果
go func() { <-altChan }()
log.Printf("[Route] %s -> %s (size: %s > %s)",
path, targetBase, FormatBytes(contentLength), FormatBytes(maxThreshold))
}
} else { } else {
// 记录没有匹配扩展名映射的情况 // 记录没有匹配扩展名映射的情况
log.Printf("[Route] %s -> %s (no extension mapping)", path, targetBase) log.Printf("[Route] %s -> %s (no extension mapping)", path, targetBase)
@ -238,6 +277,15 @@ func GetTargetURL(client *http.Client, r *http.Request, pathConfig config.PathCo
// isTargetAccessible 检查目标URL是否可访问 // isTargetAccessible 检查目标URL是否可访问
func isTargetAccessible(client *http.Client, url string) bool { func isTargetAccessible(client *http.Client, url string) bool {
// 先查缓存
if cache, ok := accessCache.Load(url); ok {
cacheItem := cache.(accessibilityCache)
if time.Since(cacheItem.timestamp) < accessTTL {
return cacheItem.accessible
}
accessCache.Delete(url)
}
req, err := http.NewRequest("HEAD", url, nil) req, err := http.NewRequest("HEAD", url, nil)
if err != nil { if err != nil {
log.Printf("[Check] Failed to create request for %s: %v", url, err) log.Printf("[Check] Failed to create request for %s: %v", url, err)
@ -255,8 +303,14 @@ func isTargetAccessible(client *http.Client, url string) bool {
} }
defer resp.Body.Close() defer resp.Body.Close()
// 检查状态码是否表示成功 accessible := resp.StatusCode >= 200 && resp.StatusCode < 400
return resp.StatusCode >= 200 && resp.StatusCode < 400 // 缓存结果
accessCache.Store(url, accessibilityCache{
accessible: accessible,
timestamp: time.Now(),
})
return accessible
} }
// SafeInt64 安全地将 interface{} 转换为 int64 // SafeInt64 安全地将 interface{} 转换为 int64

View File

@ -40,7 +40,8 @@ import {
interface PathMapping { interface PathMapping {
DefaultTarget: string DefaultTarget: string
ExtensionMap?: Record<string, string> ExtensionMap?: Record<string, string>
SizeThreshold?: number SizeThreshold?: number // 最小文件大小阈值
MaxSize?: number // 最大文件大小阈值
} }
interface FixedPath { interface FixedPath {
@ -80,6 +81,7 @@ export default function ConfigPage() {
defaultTarget: "", defaultTarget: "",
extensionMap: {} as Record<string, string>, extensionMap: {} as Record<string, string>,
sizeThreshold: 0, sizeThreshold: 0,
maxSize: 0,
sizeUnit: 'MB' as 'B' | 'KB' | 'MB' | 'GB', sizeUnit: 'MB' as 'B' | 'KB' | 'MB' | 'GB',
}) })
const [fixedPathDialogOpen, setFixedPathDialogOpen] = useState(false) const [fixedPathDialogOpen, setFixedPathDialogOpen] = useState(false)
@ -98,6 +100,7 @@ export default function ConfigPage() {
path: string; path: string;
defaultTarget: string; defaultTarget: string;
sizeThreshold: number; sizeThreshold: number;
maxSize: number;
sizeUnit: 'B' | 'KB' | 'MB' | 'GB'; sizeUnit: 'B' | 'KB' | 'MB' | 'GB';
} | null>(null); } | null>(null);
@ -218,6 +221,7 @@ export default function ConfigPage() {
defaultTarget: "", defaultTarget: "",
extensionMap: {}, extensionMap: {},
sizeThreshold: 0, sizeThreshold: 0,
maxSize: 0,
sizeUnit: 'MB', sizeUnit: 'MB',
}) })
} }
@ -251,10 +255,11 @@ export default function ConfigPage() {
const addOrUpdatePath = () => { const addOrUpdatePath = () => {
if (!config) return if (!config) return
const data = editingPathData ? editingPathData : newPathData
// 验证输入 const data = editingPathData || newPathData
if (!data.path.trim() || !data.defaultTarget.trim()) { const { path, defaultTarget, sizeThreshold, maxSize, sizeUnit } = data
if (!path || !defaultTarget) {
toast({ toast({
title: "错误", title: "错误",
description: "路径和默认目标不能为空", description: "路径和默认目标不能为空",
@ -263,66 +268,55 @@ export default function ConfigPage() {
return return
} }
// 验证路径格式 // 转换大小为字节
if (!data.path.startsWith('/')) { const sizeThresholdBytes = convertToBytes(sizeThreshold, sizeUnit)
const maxSizeBytes = convertToBytes(maxSize, sizeUnit)
// 验证阈值
if (maxSizeBytes > 0 && sizeThresholdBytes >= maxSizeBytes) {
toast({ toast({
title: "错误", title: "错误",
description: "路径必须以/开头", description: "最大文件大小阈值必须大于最小文件大小阈值",
variant: "destructive", variant: "destructive",
}) })
return return
} }
// 验证URL格式
try {
new URL(data.defaultTarget)
} catch {
toast({
title: "错误",
description: "默认目标URL格式不正确",
variant: "destructive",
})
return
}
const sizeInBytes = convertToBytes(data.sizeThreshold, data.sizeUnit)
const newConfig = { ...config } const newConfig = { ...config }
const existingMapping = newConfig.MAP[data.path] const pathConfig: PathMapping = {
DefaultTarget: defaultTarget,
ExtensionMap: {},
SizeThreshold: sizeThresholdBytes,
MaxSize: maxSizeBytes
}
// 如果是编辑现有路径,保留原有的扩展名映射
if (editingPathData && typeof config.MAP[path] === 'object') {
const existingConfig = config.MAP[path] as PathMapping
pathConfig.ExtensionMap = existingConfig.ExtensionMap
}
newConfig.MAP[path] = pathConfig
setConfig(newConfig)
if (editingPathData) { if (editingPathData) {
if (typeof existingMapping === 'object') {
newConfig.MAP[data.path] = {
DefaultTarget: data.defaultTarget,
SizeThreshold: sizeInBytes,
ExtensionMap: existingMapping.ExtensionMap || {}
}
} else {
newConfig.MAP[data.path] = {
DefaultTarget: data.defaultTarget,
SizeThreshold: sizeInBytes
}
}
} else {
newConfig.MAP[data.path] = {
DefaultTarget: data.defaultTarget,
SizeThreshold: sizeInBytes
}
}
setConfig(newConfig)
setPathDialogOpen(false)
setEditingPathData(null) setEditingPathData(null)
} else {
setNewPathData({ setNewPathData({
path: "", path: "",
defaultTarget: "", defaultTarget: "",
extensionMap: {}, extensionMap: {},
sizeThreshold: 0, sizeThreshold: 0,
maxSize: 0,
sizeUnit: 'MB', sizeUnit: 'MB',
}) })
}
setPathDialogOpen(false)
toast({ toast({
title: "成功", title: "成功",
description: "路径映射已更新", description: `${editingPathData ? '更新' : '添加'}路径配置成功`,
}) })
} }
@ -552,6 +546,7 @@ export default function ConfigPage() {
defaultTarget: "", defaultTarget: "",
extensionMap: {}, extensionMap: {},
sizeThreshold: 0, sizeThreshold: 0,
maxSize: 0,
sizeUnit: 'MB', sizeUnit: 'MB',
}) })
setPathDialogOpen(true) setPathDialogOpen(true)
@ -671,15 +666,18 @@ export default function ConfigPage() {
path, path,
defaultTarget: target, defaultTarget: target,
sizeThreshold: 0, sizeThreshold: 0,
maxSize: 0,
sizeUnit: 'MB' sizeUnit: 'MB'
}) })
} else { } else {
const sizeThreshold = target.SizeThreshold || 0 const sizeThreshold = target.SizeThreshold || 0
const maxSize = target.MaxSize || 0
const { value, unit } = convertBytesToUnit(sizeThreshold) const { value, unit } = convertBytesToUnit(sizeThreshold)
setEditingPathData({ setEditingPathData({
path, path,
defaultTarget: target.DefaultTarget, defaultTarget: target.DefaultTarget,
sizeThreshold: value, sizeThreshold: value,
maxSize: maxSize,
sizeUnit: unit sizeUnit: unit
}) })
} }
@ -787,31 +785,43 @@ export default function ConfigPage() {
</p> </p>
</div> </div>
<div className="space-y-2"> <div className="grid gap-4">
<Label></Label> <div className="grid gap-2">
<div className="flex items-center space-x-2"> <Label htmlFor="sizeThreshold"></Label>
<div className="flex gap-2">
<Input <Input
id="sizeThreshold"
type="number" type="number"
value={editingPathData ? editingPathData.sizeThreshold : newPathData.sizeThreshold} value={editingPathData?.sizeThreshold ?? newPathData.sizeThreshold}
onChange={(e) => { onChange={(e) => {
const value = parseInt(e.target.value) || 0
if (editingPathData) { if (editingPathData) {
setEditingPathData({ ...editingPathData, sizeThreshold: value }) setEditingPathData({
...editingPathData,
sizeThreshold: Number(e.target.value),
})
} else { } else {
setNewPathData({ ...newPathData, sizeThreshold: value }) setNewPathData({
...newPathData,
sizeThreshold: Number(e.target.value),
})
} }
}} }}
min={0}
/> />
<select <select
className="h-10 rounded-md border border-input bg-background px-3" className="w-24 rounded-md border border-input bg-background px-3"
value={editingPathData ? editingPathData.sizeUnit : newPathData.sizeUnit} value={editingPathData?.sizeUnit ?? newPathData.sizeUnit}
onChange={(e) => { onChange={(e) => {
const unit = e.target.value as 'B' | 'KB' | 'MB' | 'GB' const unit = e.target.value as 'B' | 'KB' | 'MB' | 'GB'
if (editingPathData) { if (editingPathData) {
setEditingPathData({ ...editingPathData, sizeUnit: unit }) setEditingPathData({
...editingPathData,
sizeUnit: unit,
})
} else { } else {
setNewPathData({ ...newPathData, sizeUnit: unit }) setNewPathData({
...newPathData,
sizeUnit: unit,
})
} }
}} }}
> >
@ -821,9 +831,53 @@ export default function ConfigPage() {
<option value="GB">GB</option> <option value="GB">GB</option>
</select> </select>
</div> </div>
<p className="text-sm text-muted-foreground"> </div>
使 <div className="grid gap-2">
</p> <Label htmlFor="maxSize"></Label>
<div className="flex gap-2">
<Input
id="maxSize"
type="number"
value={editingPathData?.maxSize ?? newPathData.maxSize}
onChange={(e) => {
if (editingPathData) {
setEditingPathData({
...editingPathData,
maxSize: Number(e.target.value),
})
} else {
setNewPathData({
...newPathData,
maxSize: Number(e.target.value),
})
}
}}
/>
<select
className="w-24 rounded-md border border-input bg-background px-3"
value={editingPathData?.sizeUnit ?? newPathData.sizeUnit}
onChange={(e) => {
const unit = e.target.value as 'B' | 'KB' | 'MB' | 'GB'
if (editingPathData) {
setEditingPathData({
...editingPathData,
sizeUnit: unit,
})
} else {
setNewPathData({
...newPathData,
sizeUnit: unit,
})
}
}}
>
<option value="B">B</option>
<option value="KB">KB</option>
<option value="MB">MB</option>
<option value="GB">GB</option>
</select>
</div>
</div>
</div> </div>
<Button onClick={addOrUpdatePath}> <Button onClick={addOrUpdatePath}>
{editingPathData ? "保存" : "添加"} {editingPathData ? "保存" : "添加"}