Sync README files from source repository [skip ci]
This commit is contained in:
Gitea Actions
@@ -0,0 +1,448 @@
|
||||
# SmartPool Cluster-Based Access Mapping
|
||||
|
||||
## Tổng quan
|
||||
|
||||
SmartPool sử dụng **cluster-based mapping** thay vì proxy-by-proxy mapping để quản lý quyền truy cập linh hoạt và scalable hơn.
|
||||
|
||||
## Thay đổi từ Proxy ID → Cluster
|
||||
|
||||
### Before (Proxy ID Mapping)
|
||||
|
||||
```sql
|
||||
CREATE TABLE proxy_access_mapping (
|
||||
access_token String,
|
||||
proxy_id Int32, -- ❌ Map trực tiếp với từng proxy
|
||||
priority Int16
|
||||
)
|
||||
```
|
||||
|
||||
**Nhược điểm:**
|
||||
- Phải update mapping mỗi khi thêm/xóa proxy
|
||||
- Khó quản lý khi có nhiều proxy
|
||||
- Không linh hoạt khi scale
|
||||
|
||||
### After (Cluster Mapping) ✅
|
||||
|
||||
```sql
|
||||
CREATE TABLE proxy_access_mapping (
|
||||
access_token String,
|
||||
cluster String, -- ✅ Map với cluster
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
```
|
||||
|
||||
**Ưu điểm:**
|
||||
- ✅ Thêm/xóa proxy trong cluster không cần update mapping
|
||||
- ✅ Dễ quản lý: 1 access_token → nhiều clusters
|
||||
- ✅ Scale dễ dàng: thêm proxy vào cluster có sẵn
|
||||
- ✅ Flexible: Có thể assign nhiều clusters với priority khác nhau
|
||||
|
||||
## Cách hoạt động
|
||||
|
||||
### 1. Proxy Metadata có Cluster
|
||||
|
||||
```sql
|
||||
INSERT INTO smart_pool_meta.proxy_metadata
|
||||
(id, host, port, protocol, ip_version, country, cluster, status)
|
||||
VALUES
|
||||
(1, '192.168.1.100', 8080, 'http', 'v4', 'VN', 'cluster_vn_http', 1),
|
||||
(2, '192.168.1.101', 8080, 'http', 'v4', 'VN', 'cluster_vn_http', 1),
|
||||
(3, '192.168.1.102', 8080, 'http', 'v6', 'US', 'cluster_us_httpv6', 1),
|
||||
(4, '192.168.1.103', 8080, 'socks5', 'v4', 'JP', 'cluster_jp_socks5', 1);
|
||||
```
|
||||
|
||||
### 2. Access Token map với Cluster
|
||||
|
||||
```sql
|
||||
INSERT INTO smart_pool_meta.proxy_access_mapping
|
||||
(access_token, cluster)
|
||||
VALUES
|
||||
('service_crawler_001', 'cluster_vn_http'),
|
||||
('service_crawler_001', 'cluster_us_httpv6'),
|
||||
('service_crawler_002', 'cluster_jp_socks5');
|
||||
```
|
||||
|
||||
### 3. Query tự động JOIN qua Cluster
|
||||
|
||||
```sql
|
||||
SELECT m.*
|
||||
FROM smart_pool_meta.proxy_metadata FINAL m
|
||||
INNER JOIN smart_pool_meta.proxy_access_mapping FINAL a
|
||||
ON m.cluster = a.cluster -- ✅ JOIN qua cluster
|
||||
WHERE a.access_token = 'service_crawler_001'
|
||||
AND m.status = 1
|
||||
ORDER BY m.id;
|
||||
|
||||
-- Kết quả: Tất cả proxies trong cluster_vn_http và cluster_us_httpv6
|
||||
```
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Use Case 1: Thêm proxy vào cluster
|
||||
|
||||
```sql
|
||||
-- Chỉ cần thêm proxy với cluster có sẵn
|
||||
INSERT INTO smart_pool_meta.proxy_metadata
|
||||
(id, host, port, cluster, status)
|
||||
VALUES
|
||||
(5, '192.168.1.105', 8080, 'cluster_vn_http', 1);
|
||||
|
||||
-- ✅ service_crawler_001 tự động có quyền dùng proxy này
|
||||
-- ❌ Không cần update proxy_access_mapping
|
||||
```
|
||||
|
||||
### Use Case 2: Gán nhiều clusters cho 1 access_token
|
||||
|
||||
```sql
|
||||
INSERT INTO smart_pool_meta.proxy_access_mapping
|
||||
(access_token, cluster)
|
||||
VALUES
|
||||
('service_crawler_001', 'cluster_vn_http'),
|
||||
('service_crawler_001', 'cluster_vn_socks5'),
|
||||
('service_crawler_001', 'cluster_us_httpv6');
|
||||
```
|
||||
|
||||
### Use Case 3: Chia sẻ cluster giữa nhiều services
|
||||
|
||||
```sql
|
||||
INSERT INTO smart_pool_meta.proxy_access_mapping
|
||||
(access_token, cluster)
|
||||
VALUES
|
||||
('service_crawler_001', 'cluster_shared'),
|
||||
('service_crawler_002', 'cluster_shared'),
|
||||
('service_api_003', 'cluster_shared');
|
||||
|
||||
-- ✅ Nhiều services dùng chung 1 cluster
|
||||
```
|
||||
|
||||
### Use Case 4: Cluster theo đặc tính
|
||||
|
||||
```sql
|
||||
-- Cluster by region
|
||||
'cluster_vn_*' -- Vietnam proxies
|
||||
'cluster_us_*' -- US proxies
|
||||
'cluster_jp_*' -- Japan proxies
|
||||
|
||||
-- Cluster by protocol
|
||||
'cluster_*_http' -- HTTP proxies
|
||||
'cluster_*_socks5' -- SOCKS5 proxies
|
||||
|
||||
-- Cluster by quality
|
||||
'cluster_premium' -- High quality proxies
|
||||
'cluster_standard' -- Standard proxies
|
||||
'cluster_backup' -- Backup proxies
|
||||
|
||||
-- Naming convention: cluster_{country}_{protocol}_{quality}
|
||||
'cluster_vn_http_premium'
|
||||
'cluster_us_socks5_standard'
|
||||
```
|
||||
|
||||
## API Usage
|
||||
|
||||
### Add Access Mapping (Updated)
|
||||
|
||||
**Before:**
|
||||
```bash
|
||||
POST /api/smart-pool/v1/SmartProxy/access-mapping/add?accessToken=token&proxyId=123&priority=10
|
||||
```
|
||||
|
||||
**After:**
|
||||
```bash
|
||||
POST /api/smart-pool/v1/SmartProxy/access-mapping/add?accessToken=token&cluster=cluster_vn_http
|
||||
```
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
curl -X POST "http://localhost:5000/api/smart-pool/v1/SmartProxy/access-mapping/add?accessToken=service_crawler_001&cluster=cluster_vn_http"
|
||||
```
|
||||
|
||||
### C# Code
|
||||
|
||||
```csharp
|
||||
// Add mapping
|
||||
await _metadataRepository.AddAccessMappingAsync(
|
||||
accessToken: "service_crawler_001",
|
||||
cluster: "cluster_vn_http"
|
||||
);
|
||||
```
|
||||
|
||||
## Migration từ Proxy ID → Cluster
|
||||
|
||||
### Step 1: Tạo Clusters
|
||||
|
||||
```sql
|
||||
-- Analyze existing proxies to group into clusters
|
||||
SELECT DISTINCT
|
||||
cluster,
|
||||
country,
|
||||
protocol,
|
||||
ip_version,
|
||||
count() as proxy_count
|
||||
FROM smart_pool_meta.proxy_metadata FINAL
|
||||
GROUP BY cluster, country, protocol, ip_version;
|
||||
```
|
||||
|
||||
### Step 2: Update Proxy Metadata
|
||||
|
||||
```sql
|
||||
-- Ensure all proxies have cluster assigned
|
||||
UPDATE smart_pool_meta.proxy_metadata
|
||||
SET cluster = concat('cluster_', country, '_', protocol)
|
||||
WHERE cluster = '' OR cluster IS NULL;
|
||||
```
|
||||
|
||||
### Step 3: Migrate Access Mappings
|
||||
|
||||
```sql
|
||||
-- Convert old proxy_id mappings to cluster mappings
|
||||
-- This is conceptual - adjust based on your old schema
|
||||
INSERT INTO smart_pool_meta.proxy_access_mapping_new (access_token, cluster, priority)
|
||||
SELECT DISTINCT
|
||||
old.access_token,
|
||||
p.cluster,
|
||||
old.priority
|
||||
FROM old_proxy_access_mapping old
|
||||
INNER JOIN smart_pool_meta.proxy_metadata FINAL p ON old.proxy_id = p.id;
|
||||
```
|
||||
|
||||
### Step 4: Verify
|
||||
|
||||
```sql
|
||||
-- Check mappings
|
||||
SELECT
|
||||
a.access_token,
|
||||
a.cluster,
|
||||
a.priority,
|
||||
count(DISTINCT m.id) as proxy_count
|
||||
FROM smart_pool_meta.proxy_access_mapping FINAL a
|
||||
LEFT JOIN smart_pool_meta.proxy_metadata FINAL m ON a.cluster = m.cluster
|
||||
GROUP BY a.access_token, a.cluster, a.priority
|
||||
ORDER BY a.access_token, a.priority DESC;
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Cluster Naming Convention
|
||||
|
||||
Sử dụng naming convention nhất quán:
|
||||
|
||||
```
|
||||
cluster_{region}_{protocol}_{quality}
|
||||
|
||||
Examples:
|
||||
- cluster_vn_http_premium
|
||||
- cluster_us_socks5_standard
|
||||
- cluster_global_http_backup
|
||||
```
|
||||
|
||||
### 2. Multiple Clusters per Token
|
||||
|
||||
```sql
|
||||
-- Assign multiple clusters to one access token
|
||||
INSERT INTO smart_pool_meta.proxy_access_mapping VALUES
|
||||
('service_crawler_001', 'cluster_vn_http'),
|
||||
('service_crawler_001', 'cluster_us_http'),
|
||||
('service_crawler_001', 'cluster_jp_http'),
|
||||
('service_crawler_001', 'cluster_global_http');
|
||||
|
||||
-- Strategy will automatically select best proxy from all available clusters
|
||||
```
|
||||
|
||||
### 3. Cluster Organization
|
||||
|
||||
```sql
|
||||
-- Regional clusters
|
||||
cluster_asia_http
|
||||
cluster_europe_http
|
||||
cluster_americas_http
|
||||
|
||||
-- Protocol-specific clusters
|
||||
cluster_http_residential
|
||||
cluster_socks5_datacenter
|
||||
|
||||
-- Quality tiers
|
||||
cluster_tier1_premium
|
||||
cluster_tier2_standard
|
||||
cluster_tier3_backup
|
||||
```
|
||||
|
||||
### 4. Dynamic Cluster Assignment
|
||||
|
||||
```csharp
|
||||
public async Task AssignServiceToClustersAsync(string accessToken, string region)
|
||||
{
|
||||
var clusters = new[]
|
||||
{
|
||||
$"cluster_{region}_http",
|
||||
$"cluster_{region}_socks5",
|
||||
"cluster_global_backup"
|
||||
};
|
||||
|
||||
foreach (var cluster in clusters)
|
||||
{
|
||||
await _metadataRepository.AddAccessMappingAsync(
|
||||
accessToken,
|
||||
cluster
|
||||
);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Query Examples
|
||||
|
||||
### Get all proxies for an access_token
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
m.id,
|
||||
m.host,
|
||||
m.port,
|
||||
m.cluster,
|
||||
m.protocol,
|
||||
m.country
|
||||
FROM smart_pool_meta.proxy_metadata FINAL m
|
||||
INNER JOIN smart_pool_meta.proxy_access_mapping FINAL a
|
||||
ON m.cluster = a.cluster
|
||||
WHERE a.access_token = 'service_crawler_001'
|
||||
AND m.status = 1
|
||||
ORDER BY m.id;
|
||||
```
|
||||
|
||||
### Get cluster statistics
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
a.access_token,
|
||||
a.cluster,
|
||||
count(DISTINCT m.id) as total_proxies,
|
||||
countIf(m.status = 1) as active_proxies,
|
||||
groupArray(DISTINCT m.country) as countries,
|
||||
groupArray(DISTINCT m.protocol) as protocols
|
||||
FROM smart_pool_meta.proxy_access_mapping FINAL a
|
||||
LEFT JOIN smart_pool_meta.proxy_metadata FINAL m
|
||||
ON a.cluster = m.cluster
|
||||
GROUP BY a.access_token, a.cluster
|
||||
ORDER BY a.access_token, a.cluster;
|
||||
```
|
||||
|
||||
### Get proxy distribution by cluster
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
cluster,
|
||||
count() as proxy_count,
|
||||
groupArray(DISTINCT country) as countries,
|
||||
groupArray(DISTINCT protocol) as protocols,
|
||||
countIf(status = 1) as active_count
|
||||
FROM smart_pool_meta.proxy_metadata FINAL
|
||||
GROUP BY cluster
|
||||
ORDER BY proxy_count DESC;
|
||||
```
|
||||
|
||||
## Benefits
|
||||
|
||||
### Scalability
|
||||
- ➕ Thêm 100 proxies vào cluster: **0 mapping updates**
|
||||
- ➖ Thêm 100 proxies với proxy-id mapping: **100 mapping updates**
|
||||
|
||||
### Flexibility
|
||||
- ✅ Dễ dàng thay đổi proxy pool mà không ảnh hưởng access control
|
||||
- ✅ Có thể re-organize clusters bất cứ lúc nào
|
||||
- ✅ Support multi-tenancy tốt hơn
|
||||
|
||||
### Maintenance
|
||||
- ✅ Ít records hơn trong mapping table
|
||||
- ✅ Queries đơn giản hơn
|
||||
- ✅ Dễ audit và debug
|
||||
|
||||
### Performance
|
||||
- ✅ Ít JOIN operations
|
||||
- ✅ Better index utilization
|
||||
- ✅ Faster query execution
|
||||
|
||||
## Example Scenario
|
||||
|
||||
### Scenario: Facebook Crawler Service
|
||||
|
||||
```sql
|
||||
-- 1. Tạo clusters cho Facebook crawling
|
||||
INSERT INTO smart_pool_meta.proxy_metadata VALUES
|
||||
-- Vietnam cluster (primary for Vietnamese users)
|
||||
(101, '10.0.1.1', 8080, 'http', 'v6', 'VN', 'cluster_fb_vn_primary', 1),
|
||||
(102, '10.0.1.2', 8080, 'http', 'v6', 'VN', 'cluster_fb_vn_primary', 1),
|
||||
(103, '10.0.1.3', 8080, 'http', 'v6', 'VN', 'cluster_fb_vn_primary', 1),
|
||||
|
||||
-- US cluster (secondary)
|
||||
(201, '10.0.2.1', 8080, 'http', 'v6', 'US', 'cluster_fb_us_secondary', 1),
|
||||
(202, '10.0.2.2', 8080, 'http', 'v6', 'US', 'cluster_fb_us_secondary', 1),
|
||||
|
||||
-- Global backup cluster
|
||||
(301, '10.0.3.1', 8080, 'socks5', 'v4', 'SG', 'cluster_fb_global_backup', 1);
|
||||
|
||||
-- 2. Assign clusters to crawler service
|
||||
INSERT INTO smart_pool_meta.proxy_access_mapping VALUES
|
||||
('fb_crawler_service', 'cluster_fb_vn_primary'),
|
||||
('fb_crawler_service', 'cluster_fb_us_secondary'),
|
||||
('fb_crawler_service', 'cluster_fb_global_backup');
|
||||
|
||||
-- 3. Crawler service tự động có access đến 6 proxies qua 3 clusters
|
||||
```
|
||||
|
||||
### Code Usage
|
||||
|
||||
```csharp
|
||||
// Service tự động pick proxy từ clusters được assign
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
accessToken: "fb_crawler_service",
|
||||
strategy: "least_delay",
|
||||
targetDomain: "facebook.com"
|
||||
);
|
||||
|
||||
// SmartPool tự động:
|
||||
// 1. Query clusters for access_token = "fb_crawler_service"
|
||||
// 2. Get all proxies in those clusters (6 proxies)
|
||||
// 3. Apply strategy to pick best one
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### No proxies available
|
||||
|
||||
```sql
|
||||
-- Debug: Check what clusters are assigned
|
||||
SELECT * FROM smart_pool_meta.proxy_access_mapping FINAL
|
||||
WHERE access_token = 'your-token';
|
||||
|
||||
-- Debug: Check proxies in those clusters
|
||||
SELECT m.*
|
||||
FROM smart_pool_meta.proxy_metadata FINAL m
|
||||
INNER JOIN smart_pool_meta.proxy_access_mapping FINAL a
|
||||
ON m.cluster = a.cluster
|
||||
WHERE a.access_token = 'your-token'
|
||||
AND m.status = 1;
|
||||
```
|
||||
|
||||
### Performance issues
|
||||
|
||||
```sql
|
||||
-- Add index on cluster column (automatically indexed by ORDER BY)
|
||||
-- Verify query performance
|
||||
EXPLAIN SYNTAX
|
||||
SELECT m.*
|
||||
FROM smart_pool_meta.proxy_metadata FINAL m
|
||||
INNER JOIN smart_pool_meta.proxy_access_mapping FINAL a
|
||||
ON m.cluster = a.cluster
|
||||
WHERE a.access_token = 'your-token';
|
||||
```
|
||||
|
||||
## Summary
|
||||
|
||||
Cluster-based mapping mang lại:
|
||||
|
||||
✅ **Flexibility**: Dễ dàng quản lý và re-organize
|
||||
✅ **Scalability**: Scale proxies mà không update mappings
|
||||
✅ **Performance**: Ít records, queries nhanh hơn
|
||||
✅ **Maintainability**: Đơn giản hóa operations
|
||||
✅ **Multi-tenancy**: Support tốt cho nhiều services
|
||||
|
||||
Thiết kế này phù hợp cho production systems với dynamic proxy pools và multiple services.
|
||||
Reference in New Issue
Block a user