SmartPool Usage Examples

Các ví dụ sử dụng SmartPool trong thực tế.

1. Basic Setup - Client Service

Cấu hình trong appsettings.json

{
  "SmartPool": {
    "Protocol": "Grpc",
    "Host": "smartpool-api.internal.com",
    "Port": 5001,
    "UseSecureConnection": false,
    "DefaultAccessToken": "service-crawler-001",
    "TimeoutMs": 30000
  }
}

Đăng ký trong Program.cs

using Icomm.SmartPool.Proxy.Extensions;

var builder = WebApplication.CreateBuilder(args);

// Đăng ký SmartPool client
builder.Services.AddSmartPoolClient(builder.Configuration);

// Hoặc cấu hình trực tiếp
builder.Services.AddSmartPoolClient(options =>
{
    options.Protocol = SmartPoolProtocol.Grpc;
    options.Host = "smartpool-api.internal.com";
    options.Port = 5001;
    options.DefaultAccessToken = "service-crawler-001";
});

var app = builder.Build();
app.Run();

2. Random Strategy - Crawl đơn giản

public class FacebookCrawlerService
{
    private readonly ISmartPoolClient _smartPoolClient;
    private readonly ILogger<FacebookCrawlerService> _logger;

    public FacebookCrawlerService(
        ISmartPoolClient smartPoolClient,
        ILogger<FacebookCrawlerService> logger)
    {
        _smartPoolClient = smartPoolClient;
        _logger = logger;
    }

    public async Task<string> CrawlPageAsync(string url)
    {
        // Lấy proxy ngẫu nhiên
        var proxy = await _smartPoolClient.GetProxyAsync(
            strategy: "random",
            ipVersion: "v6",
            country: "US"
        );

        if (proxy == null)
        {
            _logger.LogWarning("No proxy available");
            return null;
        }

        var handler = new HttpClientHandler { Proxy = proxy };
        var httpClient = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };

        var stopwatch = Stopwatch.StartNew();
        try
        {
            var response = await httpClient.GetAsync(url);
            stopwatch.Stop();

            // Log usage
            await _smartPoolClient.LogProxyUsageAsync(
                accessToken: "service-crawler-001",
                proxyId: GetProxyIdFromProxy(proxy), // Helper method
                targetDomain: "facebook.com",
                statusCode: (int)response.StatusCode,
                responseTimeMs: (int)stopwatch.ElapsedMilliseconds
            );

            return await response.Content.ReadAsStringAsync();
        }
        catch (Exception ex)
        {
            stopwatch.Stop();
            _logger.LogError(ex, "Failed to crawl {Url}", url);

            // Log failed usage
            await _smartPoolClient.LogProxyUsageAsync(
                accessToken: "service-crawler-001",
                proxyId: GetProxyIdFromProxy(proxy),
                targetDomain: "facebook.com",
                statusCode: 0,
                responseTimeMs: (int)stopwatch.ElapsedMilliseconds,
                errorMessage: ex.Message
            );

            return null;
        }
    }
}

3. Least Delay Strategy - Tối ưu tốc độ

public class HighSpeedCrawlerService
{
    private readonly ISmartPoolClient _smartPoolClient;

    public async Task<List<string>> CrawlMultipleUrlsAsync(List<string> urls)
    {
        var results = new List<string>();

        foreach (var url in urls)
        {
            // Lấy proxy nhanh nhất cho domain cụ thể
            var proxy = await _smartPoolClient.GetProxyAsync(
                strategy: "least_delay",
                targetDomain: ExtractDomain(url), // facebook.com, x.com, etc.
                ipVersion: "v6"
            );

            if (proxy != null)
            {
                var content = await CrawlWithProxyAsync(url, proxy);
                results.Add(content);
            }
        }

        return results;
    }

    private string ExtractDomain(string url)
    {
        var uri = new Uri(url);
        return uri.Host;
    }
}

4. Adaptive Ranking Strategy - Độ tin cậy cao

public class ReliableCrawlerService
{
    private readonly ISmartPoolClient _smartPoolClient;

    public async Task<string> CrawlWithBestProxyAsync(string url)
    {
        // Lấy proxy có success rate cao nhất
        var proxy = await _smartPoolClient.GetProxyAsync(
            strategy: "adaptive_ranking",
            targetDomain: "facebook.com",
            ipVersion: "v6",
            country: "US"
        );

        if (proxy == null)
        {
            throw new Exception("No reliable proxy available");
        }

        return await CrawlWithProxyAsync(url, proxy);
    }
}

5. Alternative Strategy - Retry với proxy khác

public class ResilientCrawlerService
{
    private readonly ISmartPoolClient _smartPoolClient;
    private const int MaxRetries = 3;

    public async Task<string> CrawlWithRetryAsync(string url)
    {
        SmartProxyServer currentProxy = null;
        Exception lastException = null;

        for (int attempt = 0; attempt < MaxRetries; attempt++)
        {
            try
            {
                if (attempt == 0)
                {
                    // Lần đầu: dùng random hoặc least_delay
                    var proxy = await _smartPoolClient.GetProxyAsync(
                        strategy: "least_delay",
                        targetDomain: "facebook.com"
                    );
                    currentProxy = await GetRawProxyInfo(proxy);
                }
                else
                {
                    // Retry: tìm proxy thay thế tương tự
                    var request = new GetProxyRequest
                    {
                        AccessToken = "service-crawler-001",
                        Strategy = "alternative",
                        RefererProxy = currentProxy, // Proxy vừa fail
                        TargetDomain = "facebook.com"
                    };

                    currentProxy = await _smartPoolClient.GetRawProxyAsync(request);
                }

                if (currentProxy == null)
                {
                    throw new Exception("No alternative proxy available");
                }

                // Thử crawl
                var result = await CrawlWithProxyAsync(url, currentProxy.ToWebProxy());
                return result;
            }
            catch (Exception ex)
            {
                lastException = ex;
                await Task.Delay(1000 * (attempt + 1)); // Exponential backoff
            }
        }

        throw new Exception($"Failed after {MaxRetries} attempts", lastException);
    }

    private async Task<SmartProxyServer> GetRawProxyInfo(IWebProxy proxy)
    {
        // Helper để lấy thông tin raw proxy từ IWebProxy
        // Implementation depends on your needs
        return null;
    }
}

6. Round Robin Strategy - Phân tải đều

public class LoadBalancedCrawlerService
{
    private readonly ISmartPoolClient _smartPoolClient;

    public async Task CrawlManyPagesAsync(List<string> urls)
    {
        // Round robin sẽ tự động phân phối đều các request
        var tasks = urls.Select(async url =>
        {
            var proxy = await _smartPoolClient.GetProxyAsync(
                strategy: "round_robin",
                country: "US"
            );

            return await CrawlWithProxyAsync(url, proxy);
        });

        await Task.WhenAll(tasks);
    }
}

7. Mixed Strategy - Kết hợp nhiều chiến lược

public class SmartCrawlerService
{
    private readonly ISmartPoolClient _smartPoolClient;
    private readonly Dictionary<string, string> _domainStrategies = new()
    {
        ["facebook.com"] = "adaptive_ranking",  // Cần độ tin cậy cao
        ["x.com"] = "least_delay",              // Cần tốc độ
        ["instagram.com"] = "round_robin"       // Phân tải đều
    };

    public async Task<string> CrawlSmartAsync(string url)
    {
        var domain = ExtractDomain(url);
        var strategy = _domainStrategies.GetValueOrDefault(domain, "random");

        var proxy = await _smartPoolClient.GetProxyAsync(
            strategy: strategy,
            targetDomain: domain,
            ipVersion: "v6"
        );

        return await CrawlWithProxyAsync(url, proxy);
    }
}

8. Logging Best Practices

public class ProxyLoggingService
{
    private readonly ISmartPoolClient _smartPoolClient;

    public async Task LogDetailedUsageAsync(
        int proxyId,
        string targetDomain,
        HttpResponseMessage response,
        TimeSpan duration,
        Exception exception = null)
    {
        var request = new ProxyUsageLogRequest
        {
            AccessToken = "service-crawler-001",
            ProxyId = proxyId,
            TargetDomain = targetDomain,
            StatusCode = exception != null ? 0 : (int)response.StatusCode,
            ResponseTimeMs = (int)duration.TotalMilliseconds,
            ErrorMessage = exception?.Message,
            RequestId = Guid.NewGuid().ToString("N")
        };

        await _smartPoolClient.LogProxyUsageAsync(request);
    }
}

9. Health Check Integration

public class SmartPoolHealthCheck : IHealthCheck
{
    private readonly ISmartPoolClient _smartPoolClient;

    public SmartPoolHealthCheck(ISmartPoolClient smartPoolClient)
    {
        _smartPoolClient = smartPoolClient;
    }

    public async Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context,
        CancellationToken cancellationToken = default)
    {
        try
        {
            var isHealthy = await _smartPoolClient.PingAsync(cancellationToken);
            
            if (isHealthy)
            {
                return HealthCheckResult.Healthy("SmartPool API is responding");
            }
            else
            {
                return HealthCheckResult.Unhealthy("SmartPool API is not responding");
            }
        }
        catch (Exception ex)
        {
            return HealthCheckResult.Unhealthy(
                "SmartPool API connection failed",
                ex);
        }
    }
}

// Đăng ký trong Program.cs
builder.Services.AddHealthChecks()
    .AddCheck<SmartPoolHealthCheck>("smartpool");

10. Dependency Injection Pattern

// Interface cho service của bạn
public interface ICrawlerService
{
    Task<string> CrawlAsync(string url);
}

// Implementation
public class CrawlerService : ICrawlerService
{
    private readonly ISmartPoolClient _smartPoolClient;
    private readonly ILogger<CrawlerService> _logger;
    private readonly IHttpClientFactory _httpClientFactory;

    public CrawlerService(
        ISmartPoolClient smartPoolClient,
        ILogger<CrawlerService> logger,
        IHttpClientFactory httpClientFactory)
    {
        _smartPoolClient = smartPoolClient;
        _logger = logger;
        _httpClientFactory = httpClientFactory;
    }

    public async Task<string> CrawlAsync(string url)
    {
        var proxy = await _smartPoolClient.GetProxyAsync(
            strategy: "adaptive_ranking",
            targetDomain: ExtractDomain(url)
        );

        if (proxy == null)
        {
            _logger.LogWarning("No proxy available for {Url}", url);
            return null;
        }

        var handler = new HttpClientHandler { Proxy = proxy };
        var httpClient = _httpClientFactory.CreateClient();
        httpClient = new HttpClient(handler);

        try
        {
            var response = await httpClient.GetAsync(url);
            return await response.Content.ReadAsStringAsync();
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Crawl failed for {Url}", url);
            return null;
        }
    }

    private string ExtractDomain(string url) => new Uri(url).Host;
}

// Đăng ký trong Program.cs
builder.Services.AddSmartPoolClient(builder.Configuration);
builder.Services.AddScoped<ICrawlerService, CrawlerService>();

11. Configuration per Environment

// appsettings.Development.json
{
  "SmartPool": {
    "Protocol": "Http",  // Dễ debug hơn
    "Host": "localhost",
    "Port": 5000
  }
}

// appsettings.Production.json
{
  "SmartPool": {
    "Protocol": "Grpc",  // Performance cao hơn
    "Host": "smartpool-api.internal.com",
    "Port": 5001,
    "UseSecureConnection": true
  }
}

12. Error Handling Pattern

public class SafeCrawlerService
{
    private readonly ISmartPoolClient _smartPoolClient;
    private readonly ILogger<SafeCrawlerService> _logger;

    public async Task<CrawlResult> CrawlSafelyAsync(string url)
    {
        try
        {
            var proxy = await _smartPoolClient.GetProxyAsync(
                strategy: "adaptive_ranking",
                targetDomain: ExtractDomain(url)
            );

            if (proxy == null)
            {
                return CrawlResult.Failed("No proxy available");
            }

            var content = await CrawlWithProxyAsync(url, proxy);
            return CrawlResult.Success(content);
        }
        catch (TaskCanceledException)
        {
            _logger.LogWarning("Crawl timeout for {Url}", url);
            return CrawlResult.Failed("Timeout");
        }
        catch (HttpRequestException ex)
        {
            _logger.LogError(ex, "HTTP error for {Url}", url);
            return CrawlResult.Failed($"HTTP Error: {ex.Message}");
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Unexpected error for {Url}", url);
            return CrawlResult.Failed($"Error: {ex.Message}");
        }
    }
}

public class CrawlResult
{
    public bool IsSuccess { get; set; }
    public string Content { get; set; }
    public string ErrorMessage { get; set; }

    public static CrawlResult Success(string content) =>
        new() { IsSuccess = true, Content = content };

    public static CrawlResult Failed(string error) =>
        new() { IsSuccess = false, ErrorMessage = error };
}

Kết luận

Các ví dụ trên minh họa cách sử dụng SmartPool trong nhiều tình huống thực tế:

✅ Basic setup và configuration
✅ 5 strategies với use cases cụ thể
✅ Retry logic với alternative strategy
✅ Health check integration
✅ Error handling patterns
✅ Logging best practices
✅ Dependency injection
✅ Multi-environment configuration

14 KiB Raw Permalink Blame History

SmartPool Usage Examples

1. Basic Setup - Client Service

Cấu hình trong appsettings.json

Đăng ký trong Program.cs

2. Random Strategy - Crawl đơn giản

3. Least Delay Strategy - Tối ưu tốc độ

4. Adaptive Ranking Strategy - Độ tin cậy cao

5. Alternative Strategy - Retry với proxy khác

6. Round Robin Strategy - Phân tải đều

7. Mixed Strategy - Kết hợp nhiều chiến lược

8. Logging Best Practices

9. Health Check Integration

10. Dependency Injection Pattern

11. Configuration per Environment

12. Error Handling Pattern

Kết luận

14 KiB

Raw Permalink Blame History