refactor(cache): 优化Redis键扫描与连接配置
- 使用SCAN代替KEYS扫描Redis键,避免内存占用过大 - 在批量生成随机ID时分批处理,减少Redis压力 - 更新Redis连接配置,增加连接池及超时设置 - 优化获取指定前缀键数量函数,改用SCAN实现 - 删除重复的重新获取键操作,简化逻辑流程
This commit is contained in:
38
internal/cache/redis.go
vendored
38
internal/cache/redis.go
vendored
@@ -40,9 +40,17 @@ type RedisClient struct {
|
||||
// NewRedisClient 创建新的 Redis 客户端实例
|
||||
func NewRedisClient(addr, password string, db int) (*RedisClient, error) {
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: addr,
|
||||
Password: password,
|
||||
DB: db,
|
||||
Addr: addr,
|
||||
Password: password,
|
||||
DB: db,
|
||||
PoolSize: 50, // 连接池大小
|
||||
MinIdleConns: 10, // 最小空闲连接数
|
||||
MaxRetries: 3, // 最大重试次数
|
||||
DialTimeout: 5 * time.Second, // 连接超时
|
||||
ReadTimeout: 3 * time.Second, // 读取超时
|
||||
WriteTimeout: 3 * time.Second, // 写入超时
|
||||
PoolTimeout: 4 * time.Second, // 连接池获取超时
|
||||
ConnMaxIdleTime: 5 * time.Minute, // 连接最大空闲时间
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -81,13 +89,27 @@ func (r *RedisClient) GetSize(ctx context.Context, key string) (int64, error) {
|
||||
return r.Client.DBSize(ctx).Result()
|
||||
}
|
||||
|
||||
// GetSizeByPrefix 获取以 prefix 为前缀的键值数量
|
||||
// GetSizeByPrefix 获取以 prefix 为前缀的键值数量(使用SCAN避免内存问题)
|
||||
func (r *RedisClient) GetSizeByPrefix(ctx context.Context, prefix string) (int64, error) {
|
||||
keys, err := r.Client.Keys(ctx, prefix+"*").Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
var count int64
|
||||
var cursor uint64
|
||||
var err error
|
||||
|
||||
for {
|
||||
var keys []string
|
||||
keys, cursor, err = r.Client.Scan(ctx, cursor, prefix+"*", 100).Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
count += int64(len(keys))
|
||||
|
||||
// 如果扫描完毕,退出循环
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return int64(len(keys)), nil
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// Exists 检查键是否存在
|
||||
|
||||
@@ -42,27 +42,56 @@ func (s *SendCardTaskTypeNuclear) getRandomId(ctx context.Context) (string, stri
|
||||
defer s.mu.Unlock()
|
||||
redisClient := cache.GetRedisClient()
|
||||
|
||||
// 检查是否已经有足够的ID在Redis中
|
||||
keys, err := redisClient.Client.Keys(ctx, "nuclear_random_ids:*").Result()
|
||||
if err != nil {
|
||||
otelTrace.Logger.WithContext(ctx).Error("Redis keys error", zap.Error(err))
|
||||
// 使用SCAN代替KEYS避免内存问题
|
||||
var keys []string
|
||||
var cursor uint64
|
||||
var err error
|
||||
|
||||
// 使用SCAN分批获取keys,避免内存占用过大
|
||||
for {
|
||||
var batchKeys []string
|
||||
batchKeys, cursor, err = redisClient.Client.Scan(ctx, cursor, "nuclear_random_ids:*", 100).Result()
|
||||
if err != nil {
|
||||
otelTrace.Logger.WithContext(ctx).Error("Redis scan error", zap.Error(err))
|
||||
break
|
||||
}
|
||||
keys = append(keys, batchKeys...)
|
||||
|
||||
// 如果扫描完毕,退出循环
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// 如果已经获取了足够的keys,提前退出
|
||||
if len(keys) >= 2000 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(keys) < 2000 {
|
||||
// 批量生成2k个随机数,分批处理以减少Redis压力
|
||||
batchSize := 100
|
||||
totalToGenerate := 2000
|
||||
totalToGenerate := 2000 - len(keys)
|
||||
|
||||
for batch := 0; batch < totalToGenerate/batchSize; batch++ {
|
||||
for batch := 0; batch < (totalToGenerate+batchSize-1)/batchSize; batch++ {
|
||||
// 使用Pipeline批量设置
|
||||
pipe := redisClient.Client.Pipeline()
|
||||
|
||||
for i := 0; i < batchSize; i++ {
|
||||
currentBatchSize := batchSize
|
||||
if batch == (totalToGenerate+batchSize-1)/batchSize-1 {
|
||||
currentBatchSize = totalToGenerate % batchSize
|
||||
if currentBatchSize == 0 {
|
||||
currentBatchSize = batchSize
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < currentBatchSize; i++ {
|
||||
nuclearRandomId := utils.GetMd5Lower(utils.GenerateId())
|
||||
fingerprintHash := fingerprint.GenerateRandomBrowserFingerprintHash()
|
||||
ttl := time.Hour * time.Duration(rand.Uint64N(24)+1)
|
||||
|
||||
pipe.Set(ctx, "nuclear_random_ids:"+nuclearRandomId, fingerprintHash, ttl)
|
||||
keys = append(keys, "nuclear_random_ids:"+nuclearRandomId)
|
||||
}
|
||||
|
||||
// 执行批量操作
|
||||
@@ -76,11 +105,6 @@ func (s *SendCardTaskTypeNuclear) getRandomId(ctx context.Context) (string, stri
|
||||
// 短暂延迟,避免给Redis造成过大压力
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
// 重新获取keys
|
||||
keys, err = redisClient.Client.Keys(ctx, "nuclear_random_ids:*").Result()
|
||||
if err != nil {
|
||||
otelTrace.Logger.WithContext(ctx).Error("Redis keys error after generation", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
if len(keys) == 0 {
|
||||
|
||||
Reference in New Issue
Block a user