random-api-go/service/api_fetcher.go

package service

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"random-api-go/model"
	"strings"
	"time"
)

// APIFetcher API接口获取器
type APIFetcher struct {
	client *http.Client
}

// NewAPIFetcher 创建API接口获取器
func NewAPIFetcher() *APIFetcher {
	return &APIFetcher{
		client: &http.Client{
			Timeout: 30 * time.Second,
		},
	}
}

// FetchURLs 从API接口获取URL列表
func (af *APIFetcher) FetchURLs(config *model.APIConfig) ([]string, error) {
	var allURLs []string

	// 对于GET/POST接口，我们预获取多次以获得不同的URL
	maxFetches := 200
	if config.Method == "GET" {
		maxFetches = 100 // GET接口可能返回相同结果，减少请求次数
	}

	log.Printf("开始从 %s 接口预获取 %d 次URL", config.Method, maxFetches)

	urlSet := make(map[string]bool) // 用于去重

	for i := 0; i < maxFetches; i++ {
		urls, err := af.fetchSingleRequest(config)
		if err != nil {
			log.Printf("第 %d 次请求失败: %v", i+1, err)
			continue
		}

		// 添加到集合中（自动去重）
		for _, url := range urls {
			if url != "" && !urlSet[url] {
				urlSet[url] = true
				allURLs = append(allURLs, url)
			}
		}

		// 如果是GET接口且连续几次都没有新URL，提前结束
		if config.Method == "GET" && i > 10 && len(allURLs) > 0 {
			// 检查最近10次是否有新增URL
			if i%10 == 0 {
				currentCount := len(allURLs)
				// 如果URL数量没有显著增长，可能接口返回固定结果
				if currentCount < i/5 { // 如果平均每5次请求才有1个新URL，可能效率太低
					log.Printf("GET接口效率较低，在第 %d 次请求后停止预获取", i+1)
					break
				}
			}
		}

		// 添加小延迟避免请求过快
		if i < maxFetches-1 {
			time.Sleep(50 * time.Millisecond)
		}

		// 每50次请求输出一次进度
		if (i+1)%50 == 0 {
			log.Printf("已完成 %d/%d 次请求，获得 %d 个唯一URL", i+1, maxFetches, len(allURLs))
		}
	}

	log.Printf("完成API预获取: 总共获得 %d 个唯一URL", len(allURLs))
	return allURLs, nil
}

// FetchSingleURL 实时获取单个URL (用于GET/POST实时请求)
func (af *APIFetcher) FetchSingleURL(config *model.APIConfig) ([]string, error) {
	log.Printf("实时请求 %s 接口: %s", config.Method, config.URL)
	return af.fetchSingleRequest(config)
}

// fetchSingleRequest 执行单次API请求
func (af *APIFetcher) fetchSingleRequest(config *model.APIConfig) ([]string, error) {
	var req *http.Request
	var err error

	if config.Method == "POST" {
		var body io.Reader
		if config.Body != "" {
			body = strings.NewReader(config.Body)
		}
		req, err = http.NewRequest("POST", config.URL, body)
		if config.Body != "" {
			req.Header.Set("Content-Type", "application/json")
		}
	} else {
		req, err = http.NewRequest("GET", config.URL, nil)
	}

	if err != nil {
		return nil, err
	}

	// 设置请求头
	for key, value := range config.Headers {
		req.Header.Set(key, value)
	}

	resp, err := af.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("API returned status code: %d", resp.StatusCode)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var data interface{}
	if err := json.Unmarshal(body, &data); err != nil {
		return nil, fmt.Errorf("failed to parse JSON response: %w", err)
	}

	return af.extractURLsFromJSON(data, config.URLField)
}

// extractURLsFromJSON 从JSON数据中提取URL
func (af *APIFetcher) extractURLsFromJSON(data interface{}, fieldPath string) ([]string, error) {
	var urls []string

	// 分割字段路径
	fields := strings.Split(fieldPath, ".")

	// 递归提取URL
	af.extractURLsRecursive(data, fields, 0, &urls)

	return urls, nil
}

// extractURLsRecursive 递归提取URL
func (af *APIFetcher) extractURLsRecursive(data interface{}, fields []string, depth int, urls *[]string) {
	if depth >= len(fields) {
		// 到达目标字段，提取URL
		if url, ok := data.(string); ok && url != "" {
			*urls = append(*urls, url)
		}
		return
	}

	currentField := fields[depth]

	switch v := data.(type) {
	case map[string]interface{}:
		if value, exists := v[currentField]; exists {
			af.extractURLsRecursive(value, fields, depth+1, urls)
		}
	case []interface{}:
		for _, item := range v {
			af.extractURLsRecursive(item, fields, depth, urls)
		}
	}
}