531 lines
14 KiB
Markdown
531 lines
14 KiB
Markdown
# SmartPool Usage Examples
|
|
|
|
Các ví dụ sử dụng SmartPool trong thực tế.
|
|
|
|
## 1. Basic Setup - Client Service
|
|
|
|
### Cấu hình trong appsettings.json
|
|
|
|
```json
|
|
{
|
|
"SmartPool": {
|
|
"Protocol": "Grpc",
|
|
"Host": "smartpool-api.internal.com",
|
|
"Port": 5001,
|
|
"UseSecureConnection": false,
|
|
"DefaultAccessToken": "service-crawler-001",
|
|
"TimeoutMs": 30000
|
|
}
|
|
}
|
|
```
|
|
|
|
### Đăng ký trong Program.cs
|
|
|
|
```csharp
|
|
using Icomm.SmartPool.Proxy.Extensions;
|
|
|
|
var builder = WebApplication.CreateBuilder(args);
|
|
|
|
// Đăng ký SmartPool client
|
|
builder.Services.AddSmartPoolClient(builder.Configuration);
|
|
|
|
// Hoặc cấu hình trực tiếp
|
|
builder.Services.AddSmartPoolClient(options =>
|
|
{
|
|
options.Protocol = SmartPoolProtocol.Grpc;
|
|
options.Host = "smartpool-api.internal.com";
|
|
options.Port = 5001;
|
|
options.DefaultAccessToken = "service-crawler-001";
|
|
});
|
|
|
|
var app = builder.Build();
|
|
app.Run();
|
|
```
|
|
|
|
## 2. Random Strategy - Crawl đơn giản
|
|
|
|
```csharp
|
|
public class FacebookCrawlerService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
private readonly ILogger<FacebookCrawlerService> _logger;
|
|
|
|
public FacebookCrawlerService(
|
|
ISmartPoolClient smartPoolClient,
|
|
ILogger<FacebookCrawlerService> logger)
|
|
{
|
|
_smartPoolClient = smartPoolClient;
|
|
_logger = logger;
|
|
}
|
|
|
|
public async Task<string> CrawlPageAsync(string url)
|
|
{
|
|
// Lấy proxy ngẫu nhiên
|
|
var proxy = await _smartPoolClient.GetProxyAsync(
|
|
strategy: "random",
|
|
ipVersion: "v6",
|
|
country: "US"
|
|
);
|
|
|
|
if (proxy == null)
|
|
{
|
|
_logger.LogWarning("No proxy available");
|
|
return null;
|
|
}
|
|
|
|
var handler = new HttpClientHandler { Proxy = proxy };
|
|
var httpClient = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
try
|
|
{
|
|
var response = await httpClient.GetAsync(url);
|
|
stopwatch.Stop();
|
|
|
|
// Log usage
|
|
await _smartPoolClient.LogProxyUsageAsync(
|
|
accessToken: "service-crawler-001",
|
|
proxyId: GetProxyIdFromProxy(proxy), // Helper method
|
|
targetDomain: "facebook.com",
|
|
statusCode: (int)response.StatusCode,
|
|
responseTimeMs: (int)stopwatch.ElapsedMilliseconds
|
|
);
|
|
|
|
return await response.Content.ReadAsStringAsync();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
stopwatch.Stop();
|
|
_logger.LogError(ex, "Failed to crawl {Url}", url);
|
|
|
|
// Log failed usage
|
|
await _smartPoolClient.LogProxyUsageAsync(
|
|
accessToken: "service-crawler-001",
|
|
proxyId: GetProxyIdFromProxy(proxy),
|
|
targetDomain: "facebook.com",
|
|
statusCode: 0,
|
|
responseTimeMs: (int)stopwatch.ElapsedMilliseconds,
|
|
errorMessage: ex.Message
|
|
);
|
|
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
## 3. Least Delay Strategy - Tối ưu tốc độ
|
|
|
|
```csharp
|
|
public class HighSpeedCrawlerService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
|
|
public async Task<List<string>> CrawlMultipleUrlsAsync(List<string> urls)
|
|
{
|
|
var results = new List<string>();
|
|
|
|
foreach (var url in urls)
|
|
{
|
|
// Lấy proxy nhanh nhất cho domain cụ thể
|
|
var proxy = await _smartPoolClient.GetProxyAsync(
|
|
strategy: "least_delay",
|
|
targetDomain: ExtractDomain(url), // facebook.com, x.com, etc.
|
|
ipVersion: "v6"
|
|
);
|
|
|
|
if (proxy != null)
|
|
{
|
|
var content = await CrawlWithProxyAsync(url, proxy);
|
|
results.Add(content);
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
private string ExtractDomain(string url)
|
|
{
|
|
var uri = new Uri(url);
|
|
return uri.Host;
|
|
}
|
|
}
|
|
```
|
|
|
|
## 4. Adaptive Ranking Strategy - Độ tin cậy cao
|
|
|
|
```csharp
|
|
public class ReliableCrawlerService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
|
|
public async Task<string> CrawlWithBestProxyAsync(string url)
|
|
{
|
|
// Lấy proxy có success rate cao nhất
|
|
var proxy = await _smartPoolClient.GetProxyAsync(
|
|
strategy: "adaptive_ranking",
|
|
targetDomain: "facebook.com",
|
|
ipVersion: "v6",
|
|
country: "US"
|
|
);
|
|
|
|
if (proxy == null)
|
|
{
|
|
throw new Exception("No reliable proxy available");
|
|
}
|
|
|
|
return await CrawlWithProxyAsync(url, proxy);
|
|
}
|
|
}
|
|
```
|
|
|
|
## 5. Alternative Strategy - Retry với proxy khác
|
|
|
|
```csharp
|
|
public class ResilientCrawlerService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
private const int MaxRetries = 3;
|
|
|
|
public async Task<string> CrawlWithRetryAsync(string url)
|
|
{
|
|
SmartProxyServer currentProxy = null;
|
|
Exception lastException = null;
|
|
|
|
for (int attempt = 0; attempt < MaxRetries; attempt++)
|
|
{
|
|
try
|
|
{
|
|
if (attempt == 0)
|
|
{
|
|
// Lần đầu: dùng random hoặc least_delay
|
|
var proxy = await _smartPoolClient.GetProxyAsync(
|
|
strategy: "least_delay",
|
|
targetDomain: "facebook.com"
|
|
);
|
|
currentProxy = await GetRawProxyInfo(proxy);
|
|
}
|
|
else
|
|
{
|
|
// Retry: tìm proxy thay thế tương tự
|
|
var request = new GetProxyRequest
|
|
{
|
|
AccessToken = "service-crawler-001",
|
|
Strategy = "alternative",
|
|
RefererProxy = currentProxy, // Proxy vừa fail
|
|
TargetDomain = "facebook.com"
|
|
};
|
|
|
|
currentProxy = await _smartPoolClient.GetRawProxyAsync(request);
|
|
}
|
|
|
|
if (currentProxy == null)
|
|
{
|
|
throw new Exception("No alternative proxy available");
|
|
}
|
|
|
|
// Thử crawl
|
|
var result = await CrawlWithProxyAsync(url, currentProxy.ToWebProxy());
|
|
return result;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
lastException = ex;
|
|
await Task.Delay(1000 * (attempt + 1)); // Exponential backoff
|
|
}
|
|
}
|
|
|
|
throw new Exception($"Failed after {MaxRetries} attempts", lastException);
|
|
}
|
|
|
|
private async Task<SmartProxyServer> GetRawProxyInfo(IWebProxy proxy)
|
|
{
|
|
// Helper để lấy thông tin raw proxy từ IWebProxy
|
|
// Implementation depends on your needs
|
|
return null;
|
|
}
|
|
}
|
|
```
|
|
|
|
## 6. Round Robin Strategy - Phân tải đều
|
|
|
|
```csharp
|
|
public class LoadBalancedCrawlerService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
|
|
public async Task CrawlManyPagesAsync(List<string> urls)
|
|
{
|
|
// Round robin sẽ tự động phân phối đều các request
|
|
var tasks = urls.Select(async url =>
|
|
{
|
|
var proxy = await _smartPoolClient.GetProxyAsync(
|
|
strategy: "round_robin",
|
|
country: "US"
|
|
);
|
|
|
|
return await CrawlWithProxyAsync(url, proxy);
|
|
});
|
|
|
|
await Task.WhenAll(tasks);
|
|
}
|
|
}
|
|
```
|
|
|
|
## 7. Mixed Strategy - Kết hợp nhiều chiến lược
|
|
|
|
```csharp
|
|
public class SmartCrawlerService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
private readonly Dictionary<string, string> _domainStrategies = new()
|
|
{
|
|
["facebook.com"] = "adaptive_ranking", // Cần độ tin cậy cao
|
|
["x.com"] = "least_delay", // Cần tốc độ
|
|
["instagram.com"] = "round_robin" // Phân tải đều
|
|
};
|
|
|
|
public async Task<string> CrawlSmartAsync(string url)
|
|
{
|
|
var domain = ExtractDomain(url);
|
|
var strategy = _domainStrategies.GetValueOrDefault(domain, "random");
|
|
|
|
var proxy = await _smartPoolClient.GetProxyAsync(
|
|
strategy: strategy,
|
|
targetDomain: domain,
|
|
ipVersion: "v6"
|
|
);
|
|
|
|
return await CrawlWithProxyAsync(url, proxy);
|
|
}
|
|
}
|
|
```
|
|
|
|
## 8. Logging Best Practices
|
|
|
|
```csharp
|
|
public class ProxyLoggingService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
|
|
public async Task LogDetailedUsageAsync(
|
|
int proxyId,
|
|
string targetDomain,
|
|
HttpResponseMessage response,
|
|
TimeSpan duration,
|
|
Exception exception = null)
|
|
{
|
|
var request = new ProxyUsageLogRequest
|
|
{
|
|
AccessToken = "service-crawler-001",
|
|
ProxyId = proxyId,
|
|
TargetDomain = targetDomain,
|
|
StatusCode = exception != null ? 0 : (int)response.StatusCode,
|
|
ResponseTimeMs = (int)duration.TotalMilliseconds,
|
|
ErrorMessage = exception?.Message,
|
|
RequestId = Guid.NewGuid().ToString("N")
|
|
};
|
|
|
|
await _smartPoolClient.LogProxyUsageAsync(request);
|
|
}
|
|
}
|
|
```
|
|
|
|
## 9. Health Check Integration
|
|
|
|
```csharp
|
|
public class SmartPoolHealthCheck : IHealthCheck
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
|
|
public SmartPoolHealthCheck(ISmartPoolClient smartPoolClient)
|
|
{
|
|
_smartPoolClient = smartPoolClient;
|
|
}
|
|
|
|
public async Task<HealthCheckResult> CheckHealthAsync(
|
|
HealthCheckContext context,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
try
|
|
{
|
|
var isHealthy = await _smartPoolClient.PingAsync(cancellationToken);
|
|
|
|
if (isHealthy)
|
|
{
|
|
return HealthCheckResult.Healthy("SmartPool API is responding");
|
|
}
|
|
else
|
|
{
|
|
return HealthCheckResult.Unhealthy("SmartPool API is not responding");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return HealthCheckResult.Unhealthy(
|
|
"SmartPool API connection failed",
|
|
ex);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Đăng ký trong Program.cs
|
|
builder.Services.AddHealthChecks()
|
|
.AddCheck<SmartPoolHealthCheck>("smartpool");
|
|
```
|
|
|
|
## 10. Dependency Injection Pattern
|
|
|
|
```csharp
|
|
// Interface cho service của bạn
|
|
public interface ICrawlerService
|
|
{
|
|
Task<string> CrawlAsync(string url);
|
|
}
|
|
|
|
// Implementation
|
|
public class CrawlerService : ICrawlerService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
private readonly ILogger<CrawlerService> _logger;
|
|
private readonly IHttpClientFactory _httpClientFactory;
|
|
|
|
public CrawlerService(
|
|
ISmartPoolClient smartPoolClient,
|
|
ILogger<CrawlerService> logger,
|
|
IHttpClientFactory httpClientFactory)
|
|
{
|
|
_smartPoolClient = smartPoolClient;
|
|
_logger = logger;
|
|
_httpClientFactory = httpClientFactory;
|
|
}
|
|
|
|
public async Task<string> CrawlAsync(string url)
|
|
{
|
|
var proxy = await _smartPoolClient.GetProxyAsync(
|
|
strategy: "adaptive_ranking",
|
|
targetDomain: ExtractDomain(url)
|
|
);
|
|
|
|
if (proxy == null)
|
|
{
|
|
_logger.LogWarning("No proxy available for {Url}", url);
|
|
return null;
|
|
}
|
|
|
|
var handler = new HttpClientHandler { Proxy = proxy };
|
|
var httpClient = _httpClientFactory.CreateClient();
|
|
httpClient = new HttpClient(handler);
|
|
|
|
try
|
|
{
|
|
var response = await httpClient.GetAsync(url);
|
|
return await response.Content.ReadAsStringAsync();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Crawl failed for {Url}", url);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private string ExtractDomain(string url) => new Uri(url).Host;
|
|
}
|
|
|
|
// Đăng ký trong Program.cs
|
|
builder.Services.AddSmartPoolClient(builder.Configuration);
|
|
builder.Services.AddScoped<ICrawlerService, CrawlerService>();
|
|
```
|
|
|
|
## 11. Configuration per Environment
|
|
|
|
```csharp
|
|
// appsettings.Development.json
|
|
{
|
|
"SmartPool": {
|
|
"Protocol": "Http", // Dễ debug hơn
|
|
"Host": "localhost",
|
|
"Port": 5000
|
|
}
|
|
}
|
|
|
|
// appsettings.Production.json
|
|
{
|
|
"SmartPool": {
|
|
"Protocol": "Grpc", // Performance cao hơn
|
|
"Host": "smartpool-api.internal.com",
|
|
"Port": 5001,
|
|
"UseSecureConnection": true
|
|
}
|
|
}
|
|
```
|
|
|
|
## 12. Error Handling Pattern
|
|
|
|
```csharp
|
|
public class SafeCrawlerService
|
|
{
|
|
private readonly ISmartPoolClient _smartPoolClient;
|
|
private readonly ILogger<SafeCrawlerService> _logger;
|
|
|
|
public async Task<CrawlResult> CrawlSafelyAsync(string url)
|
|
{
|
|
try
|
|
{
|
|
var proxy = await _smartPoolClient.GetProxyAsync(
|
|
strategy: "adaptive_ranking",
|
|
targetDomain: ExtractDomain(url)
|
|
);
|
|
|
|
if (proxy == null)
|
|
{
|
|
return CrawlResult.Failed("No proxy available");
|
|
}
|
|
|
|
var content = await CrawlWithProxyAsync(url, proxy);
|
|
return CrawlResult.Success(content);
|
|
}
|
|
catch (TaskCanceledException)
|
|
{
|
|
_logger.LogWarning("Crawl timeout for {Url}", url);
|
|
return CrawlResult.Failed("Timeout");
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
_logger.LogError(ex, "HTTP error for {Url}", url);
|
|
return CrawlResult.Failed($"HTTP Error: {ex.Message}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Unexpected error for {Url}", url);
|
|
return CrawlResult.Failed($"Error: {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
|
|
public class CrawlResult
|
|
{
|
|
public bool IsSuccess { get; set; }
|
|
public string Content { get; set; }
|
|
public string ErrorMessage { get; set; }
|
|
|
|
public static CrawlResult Success(string content) =>
|
|
new() { IsSuccess = true, Content = content };
|
|
|
|
public static CrawlResult Failed(string error) =>
|
|
new() { IsSuccess = false, ErrorMessage = error };
|
|
}
|
|
```
|
|
|
|
## Kết luận
|
|
|
|
Các ví dụ trên minh họa cách sử dụng SmartPool trong nhiều tình huống thực tế:
|
|
- ✅ Basic setup và configuration
|
|
- ✅ 5 strategies với use cases cụ thể
|
|
- ✅ Retry logic với alternative strategy
|
|
- ✅ Health check integration
|
|
- ✅ Error handling patterns
|
|
- ✅ Logging best practices
|
|
- ✅ Dependency injection
|
|
- ✅ Multi-environment configuration
|