Sync README files from source repository [skip ci]
This commit is contained in:
Gitea Actions
@@ -0,0 +1,530 @@
|
||||
# SmartPool Usage Examples
|
||||
|
||||
Các ví dụ sử dụng SmartPool trong thực tế.
|
||||
|
||||
## 1. Basic Setup - Client Service
|
||||
|
||||
### Cấu hình trong appsettings.json
|
||||
|
||||
```json
|
||||
{
|
||||
"SmartPool": {
|
||||
"Protocol": "Grpc",
|
||||
"Host": "smartpool-api.internal.com",
|
||||
"Port": 5001,
|
||||
"UseSecureConnection": false,
|
||||
"DefaultAccessToken": "service-crawler-001",
|
||||
"TimeoutMs": 30000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Đăng ký trong Program.cs
|
||||
|
||||
```csharp
|
||||
using Icomm.SmartPool.Proxy.Extensions;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
// Đăng ký SmartPool client
|
||||
builder.Services.AddSmartPoolClient(builder.Configuration);
|
||||
|
||||
// Hoặc cấu hình trực tiếp
|
||||
builder.Services.AddSmartPoolClient(options =>
|
||||
{
|
||||
options.Protocol = SmartPoolProtocol.Grpc;
|
||||
options.Host = "smartpool-api.internal.com";
|
||||
options.Port = 5001;
|
||||
options.DefaultAccessToken = "service-crawler-001";
|
||||
});
|
||||
|
||||
var app = builder.Build();
|
||||
app.Run();
|
||||
```
|
||||
|
||||
## 2. Random Strategy - Crawl đơn giản
|
||||
|
||||
```csharp
|
||||
public class FacebookCrawlerService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
private readonly ILogger<FacebookCrawlerService> _logger;
|
||||
|
||||
public FacebookCrawlerService(
|
||||
ISmartPoolClient smartPoolClient,
|
||||
ILogger<FacebookCrawlerService> logger)
|
||||
{
|
||||
_smartPoolClient = smartPoolClient;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<string> CrawlPageAsync(string url)
|
||||
{
|
||||
// Lấy proxy ngẫu nhiên
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
strategy: "random",
|
||||
ipVersion: "v6",
|
||||
country: "US"
|
||||
);
|
||||
|
||||
if (proxy == null)
|
||||
{
|
||||
_logger.LogWarning("No proxy available");
|
||||
return null;
|
||||
}
|
||||
|
||||
var handler = new HttpClientHandler { Proxy = proxy };
|
||||
var httpClient = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var response = await httpClient.GetAsync(url);
|
||||
stopwatch.Stop();
|
||||
|
||||
// Log usage
|
||||
await _smartPoolClient.LogProxyUsageAsync(
|
||||
accessToken: "service-crawler-001",
|
||||
proxyId: GetProxyIdFromProxy(proxy), // Helper method
|
||||
targetDomain: "facebook.com",
|
||||
statusCode: (int)response.StatusCode,
|
||||
responseTimeMs: (int)stopwatch.ElapsedMilliseconds
|
||||
);
|
||||
|
||||
return await response.Content.ReadAsStringAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_logger.LogError(ex, "Failed to crawl {Url}", url);
|
||||
|
||||
// Log failed usage
|
||||
await _smartPoolClient.LogProxyUsageAsync(
|
||||
accessToken: "service-crawler-001",
|
||||
proxyId: GetProxyIdFromProxy(proxy),
|
||||
targetDomain: "facebook.com",
|
||||
statusCode: 0,
|
||||
responseTimeMs: (int)stopwatch.ElapsedMilliseconds,
|
||||
errorMessage: ex.Message
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 3. Least Delay Strategy - Tối ưu tốc độ
|
||||
|
||||
```csharp
|
||||
public class HighSpeedCrawlerService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
|
||||
public async Task<List<string>> CrawlMultipleUrlsAsync(List<string> urls)
|
||||
{
|
||||
var results = new List<string>();
|
||||
|
||||
foreach (var url in urls)
|
||||
{
|
||||
// Lấy proxy nhanh nhất cho domain cụ thể
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
strategy: "least_delay",
|
||||
targetDomain: ExtractDomain(url), // facebook.com, x.com, etc.
|
||||
ipVersion: "v6"
|
||||
);
|
||||
|
||||
if (proxy != null)
|
||||
{
|
||||
var content = await CrawlWithProxyAsync(url, proxy);
|
||||
results.Add(content);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private string ExtractDomain(string url)
|
||||
{
|
||||
var uri = new Uri(url);
|
||||
return uri.Host;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 4. Adaptive Ranking Strategy - Độ tin cậy cao
|
||||
|
||||
```csharp
|
||||
public class ReliableCrawlerService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
|
||||
public async Task<string> CrawlWithBestProxyAsync(string url)
|
||||
{
|
||||
// Lấy proxy có success rate cao nhất
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
strategy: "adaptive_ranking",
|
||||
targetDomain: "facebook.com",
|
||||
ipVersion: "v6",
|
||||
country: "US"
|
||||
);
|
||||
|
||||
if (proxy == null)
|
||||
{
|
||||
throw new Exception("No reliable proxy available");
|
||||
}
|
||||
|
||||
return await CrawlWithProxyAsync(url, proxy);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 5. Alternative Strategy - Retry với proxy khác
|
||||
|
||||
```csharp
|
||||
public class ResilientCrawlerService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
private const int MaxRetries = 3;
|
||||
|
||||
public async Task<string> CrawlWithRetryAsync(string url)
|
||||
{
|
||||
SmartProxyServer currentProxy = null;
|
||||
Exception lastException = null;
|
||||
|
||||
for (int attempt = 0; attempt < MaxRetries; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (attempt == 0)
|
||||
{
|
||||
// Lần đầu: dùng random hoặc least_delay
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
strategy: "least_delay",
|
||||
targetDomain: "facebook.com"
|
||||
);
|
||||
currentProxy = await GetRawProxyInfo(proxy);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Retry: tìm proxy thay thế tương tự
|
||||
var request = new GetProxyRequest
|
||||
{
|
||||
AccessToken = "service-crawler-001",
|
||||
Strategy = "alternative",
|
||||
RefererProxy = currentProxy, // Proxy vừa fail
|
||||
TargetDomain = "facebook.com"
|
||||
};
|
||||
|
||||
currentProxy = await _smartPoolClient.GetRawProxyAsync(request);
|
||||
}
|
||||
|
||||
if (currentProxy == null)
|
||||
{
|
||||
throw new Exception("No alternative proxy available");
|
||||
}
|
||||
|
||||
// Thử crawl
|
||||
var result = await CrawlWithProxyAsync(url, currentProxy.ToWebProxy());
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
lastException = ex;
|
||||
await Task.Delay(1000 * (attempt + 1)); // Exponential backoff
|
||||
}
|
||||
}
|
||||
|
||||
throw new Exception($"Failed after {MaxRetries} attempts", lastException);
|
||||
}
|
||||
|
||||
private async Task<SmartProxyServer> GetRawProxyInfo(IWebProxy proxy)
|
||||
{
|
||||
// Helper để lấy thông tin raw proxy từ IWebProxy
|
||||
// Implementation depends on your needs
|
||||
return null;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 6. Round Robin Strategy - Phân tải đều
|
||||
|
||||
```csharp
|
||||
public class LoadBalancedCrawlerService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
|
||||
public async Task CrawlManyPagesAsync(List<string> urls)
|
||||
{
|
||||
// Round robin sẽ tự động phân phối đều các request
|
||||
var tasks = urls.Select(async url =>
|
||||
{
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
strategy: "round_robin",
|
||||
country: "US"
|
||||
);
|
||||
|
||||
return await CrawlWithProxyAsync(url, proxy);
|
||||
});
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 7. Mixed Strategy - Kết hợp nhiều chiến lược
|
||||
|
||||
```csharp
|
||||
public class SmartCrawlerService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
private readonly Dictionary<string, string> _domainStrategies = new()
|
||||
{
|
||||
["facebook.com"] = "adaptive_ranking", // Cần độ tin cậy cao
|
||||
["x.com"] = "least_delay", // Cần tốc độ
|
||||
["instagram.com"] = "round_robin" // Phân tải đều
|
||||
};
|
||||
|
||||
public async Task<string> CrawlSmartAsync(string url)
|
||||
{
|
||||
var domain = ExtractDomain(url);
|
||||
var strategy = _domainStrategies.GetValueOrDefault(domain, "random");
|
||||
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
strategy: strategy,
|
||||
targetDomain: domain,
|
||||
ipVersion: "v6"
|
||||
);
|
||||
|
||||
return await CrawlWithProxyAsync(url, proxy);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 8. Logging Best Practices
|
||||
|
||||
```csharp
|
||||
public class ProxyLoggingService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
|
||||
public async Task LogDetailedUsageAsync(
|
||||
int proxyId,
|
||||
string targetDomain,
|
||||
HttpResponseMessage response,
|
||||
TimeSpan duration,
|
||||
Exception exception = null)
|
||||
{
|
||||
var request = new ProxyUsageLogRequest
|
||||
{
|
||||
AccessToken = "service-crawler-001",
|
||||
ProxyId = proxyId,
|
||||
TargetDomain = targetDomain,
|
||||
StatusCode = exception != null ? 0 : (int)response.StatusCode,
|
||||
ResponseTimeMs = (int)duration.TotalMilliseconds,
|
||||
ErrorMessage = exception?.Message,
|
||||
RequestId = Guid.NewGuid().ToString("N")
|
||||
};
|
||||
|
||||
await _smartPoolClient.LogProxyUsageAsync(request);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 9. Health Check Integration
|
||||
|
||||
```csharp
|
||||
public class SmartPoolHealthCheck : IHealthCheck
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
|
||||
public SmartPoolHealthCheck(ISmartPoolClient smartPoolClient)
|
||||
{
|
||||
_smartPoolClient = smartPoolClient;
|
||||
}
|
||||
|
||||
public async Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var isHealthy = await _smartPoolClient.PingAsync(cancellationToken);
|
||||
|
||||
if (isHealthy)
|
||||
{
|
||||
return HealthCheckResult.Healthy("SmartPool API is responding");
|
||||
}
|
||||
else
|
||||
{
|
||||
return HealthCheckResult.Unhealthy("SmartPool API is not responding");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return HealthCheckResult.Unhealthy(
|
||||
"SmartPool API connection failed",
|
||||
ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Đăng ký trong Program.cs
|
||||
builder.Services.AddHealthChecks()
|
||||
.AddCheck<SmartPoolHealthCheck>("smartpool");
|
||||
```
|
||||
|
||||
## 10. Dependency Injection Pattern
|
||||
|
||||
```csharp
|
||||
// Interface cho service của bạn
|
||||
public interface ICrawlerService
|
||||
{
|
||||
Task<string> CrawlAsync(string url);
|
||||
}
|
||||
|
||||
// Implementation
|
||||
public class CrawlerService : ICrawlerService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
private readonly ILogger<CrawlerService> _logger;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
|
||||
public CrawlerService(
|
||||
ISmartPoolClient smartPoolClient,
|
||||
ILogger<CrawlerService> logger,
|
||||
IHttpClientFactory httpClientFactory)
|
||||
{
|
||||
_smartPoolClient = smartPoolClient;
|
||||
_logger = logger;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
}
|
||||
|
||||
public async Task<string> CrawlAsync(string url)
|
||||
{
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
strategy: "adaptive_ranking",
|
||||
targetDomain: ExtractDomain(url)
|
||||
);
|
||||
|
||||
if (proxy == null)
|
||||
{
|
||||
_logger.LogWarning("No proxy available for {Url}", url);
|
||||
return null;
|
||||
}
|
||||
|
||||
var handler = new HttpClientHandler { Proxy = proxy };
|
||||
var httpClient = _httpClientFactory.CreateClient();
|
||||
httpClient = new HttpClient(handler);
|
||||
|
||||
try
|
||||
{
|
||||
var response = await httpClient.GetAsync(url);
|
||||
return await response.Content.ReadAsStringAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Crawl failed for {Url}", url);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private string ExtractDomain(string url) => new Uri(url).Host;
|
||||
}
|
||||
|
||||
// Đăng ký trong Program.cs
|
||||
builder.Services.AddSmartPoolClient(builder.Configuration);
|
||||
builder.Services.AddScoped<ICrawlerService, CrawlerService>();
|
||||
```
|
||||
|
||||
## 11. Configuration per Environment
|
||||
|
||||
```csharp
|
||||
// appsettings.Development.json
|
||||
{
|
||||
"SmartPool": {
|
||||
"Protocol": "Http", // Dễ debug hơn
|
||||
"Host": "localhost",
|
||||
"Port": 5000
|
||||
}
|
||||
}
|
||||
|
||||
// appsettings.Production.json
|
||||
{
|
||||
"SmartPool": {
|
||||
"Protocol": "Grpc", // Performance cao hơn
|
||||
"Host": "smartpool-api.internal.com",
|
||||
"Port": 5001,
|
||||
"UseSecureConnection": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 12. Error Handling Pattern
|
||||
|
||||
```csharp
|
||||
public class SafeCrawlerService
|
||||
{
|
||||
private readonly ISmartPoolClient _smartPoolClient;
|
||||
private readonly ILogger<SafeCrawlerService> _logger;
|
||||
|
||||
public async Task<CrawlResult> CrawlSafelyAsync(string url)
|
||||
{
|
||||
try
|
||||
{
|
||||
var proxy = await _smartPoolClient.GetProxyAsync(
|
||||
strategy: "adaptive_ranking",
|
||||
targetDomain: ExtractDomain(url)
|
||||
);
|
||||
|
||||
if (proxy == null)
|
||||
{
|
||||
return CrawlResult.Failed("No proxy available");
|
||||
}
|
||||
|
||||
var content = await CrawlWithProxyAsync(url, proxy);
|
||||
return CrawlResult.Success(content);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning("Crawl timeout for {Url}", url);
|
||||
return CrawlResult.Failed("Timeout");
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
_logger.LogError(ex, "HTTP error for {Url}", url);
|
||||
return CrawlResult.Failed($"HTTP Error: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unexpected error for {Url}", url);
|
||||
return CrawlResult.Failed($"Error: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class CrawlResult
|
||||
{
|
||||
public bool IsSuccess { get; set; }
|
||||
public string Content { get; set; }
|
||||
public string ErrorMessage { get; set; }
|
||||
|
||||
public static CrawlResult Success(string content) =>
|
||||
new() { IsSuccess = true, Content = content };
|
||||
|
||||
public static CrawlResult Failed(string error) =>
|
||||
new() { IsSuccess = false, ErrorMessage = error };
|
||||
}
|
||||
```
|
||||
|
||||
## Kết luận
|
||||
|
||||
Các ví dụ trên minh họa cách sử dụng SmartPool trong nhiều tình huống thực tế:
|
||||
- ✅ Basic setup và configuration
|
||||
- ✅ 5 strategies với use cases cụ thể
|
||||
- ✅ Retry logic với alternative strategy
|
||||
- ✅ Health check integration
|
||||
- ✅ Error handling patterns
|
||||
- ✅ Logging best practices
|
||||
- ✅ Dependency injection
|
||||
- ✅ Multi-environment configuration
|
||||
Reference in New Issue
Block a user