diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..29c4713 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Generated files +docker-compose.yml +config/squid.conf +config/allowed_ip.txt +config/nginx.conf +config/basic_db_ip_auth.php + +# Composer dependencies +setup/vendor/ +setup/composer.phar + +# User configuration +setup/config.php +setup/squid.conf + +# Database and cache volumes +volumes/ + +# Temporary files +*.log +*.tmp diff --git a/README.md b/README.md index 498afda..47e6aa3 100644 --- a/README.md +++ b/README.md @@ -1,241 +1,443 @@ -# Docker Rotating Proxy Config Generator +# Docker Rotating Proxy -- Fully Optimized for Web Scraping Usage. -- HTTP/HTTPS Support (see wiki) -- socks5 with Authorization Proxy to HTTP(S) proxy convert compatible by [Gost](https://github.com/ginuerzh/gost) -- You can use a VPN as an HTTP proxy.(powered by [gluetun](https://github.com/qdm12/gluetun) ) -- Making it IP address based authentication makes it easier to use in your program.(selenium,puppeteer etc) +A flexible rotating proxy system powered by Docker, Squid, and Gost. Automatically rotates through multiple proxy sources for web scraping and privacy. +## Features +- **Multiple Proxy Types**: HTTP, HTTPS, SOCKS5, and OpenVPN +- **Flexible Format**: Customize proxy list column order and delimiters +- **Auto Rotation**: Squid automatically rotates proxies per request +- **Static IPs**: Direct port access for session persistence +- **Easy Configuration**: Simple text file configuration +- **Docker-based**: Portable and easy to deploy + +## Quick Start + +### 1. Clone Repository + +```bash +git clone https://github.com/39ff/docker-rotating-proxy +cd docker-rotating-proxy ``` - Docker Container - ---------------------------------- -Client <----> Squid <-> HTTP/HTTPS Rotate Proxies---\ - | - ---------------|-> Gost <-> Socks5 Proxy --- Internet - | - --------------<-> VPN to HTTP Proxy <--------/ - - -It can be used in two ways. -1.Automatically control the proxy and rotate each request -> use Squid -2.Control the proxy programmatically　-> use Gost Port +### 2. Install Dependencies +```bash +cd setup +docker run --rm -it -v "$(pwd):/app" composer install +cd .. ``` +### 3. Configure Proxies -## Usage Example +Edit `proxyList.txt` with your proxies. The format is flexible: -### Configuring IPs to allow the use of the Rotating Proxy -If you want to use it from outside, please specify the **your** IP address to allowed_ip.txt +**Example 1: Default format (colon-separated)** +``` +# format: host:port:scheme:user:pass +proxy1.example.com:1080:socks5:username:password +proxy2.example.com:8080:http:user:pass +192.168.1.100:3128 +``` -http://httpbin.org/ip +**Example 2: Custom format (comma-separated)** +``` +# format: host,port,scheme,user,pass +159.89.206.161,2434,socks5,vpn,unlimited +142.93.68.63,2434,socks5,vpn,unlimited +``` + +**Example 3: Custom order (scheme first)** +``` +# format: scheme|host|port|user|pass +socks5|proxy.example.com|1080|myuser|mypass +``` + +**Supported formats:** +- `host:port` - Simple HTTP proxy +- `host:port:scheme` - Proxy with protocol +- `host:port:scheme:user:pass` - With authentication +- Delimiters: `:` (colon), `,` (comma), `|` (pipe), tab, or space + +**Supported schemes:** +- `socks5` - Creates Gost bridge container +- `http` / `https` - Creates Gost bridge container +- `httpsquid` - Direct squid integration (no container) +- *(empty)* - Direct HTTP proxy (no container) + +### 4. Configure Allowed IPs + +Edit `template/allowed_ip.txt` to whitelist client IPs: -Example: ``` 93.184.216.34 108.62.57.53 ``` -### 1. Create your proxyList.txt(If HTTP/Socks is provided) -Search FreeProxy List or Paid/Subscribe ProxyService Provider. +Get your IP: http://httpbin.org/ip + +### 5. Generate Configuration + +```bash +# Remove OpenVPN examples if not needed +rm -rf ./openvpn/* + +# Generate docker-compose.yml +docker run --rm -it -v "$(pwd):/app/" php:8.2-cli php /app/setup/generate.php +``` -example : https://github.com/clarketm/proxy-list +### 6. Start Services -#### Format +```bash +docker-compose up -d ``` -IPAddress:Port:Type(socks5 or http or https or httpsquid):Username:Password -IPAddress:Port:Type(socks5 or http or https or httpsquid) -IPAddress:Port + +### 7. Test + +```bash +# Rotating proxy (port 3128) +curl http://httpbin.org/ip --proxy http://127.0.0.1:3128 + +# Static proxy (port 30000+) +curl http://httpbin.org/ip --proxy http://127.0.0.1:30000 ``` -### 1.1 Create Your OpenVPN Config(If HTTP/Socks is NOT provided) -see [example](openvpn/) +## Usage Modes + +### Mode 1: Rotating Proxy (Port 3128) + +Squid automatically rotates through all configured proxies. Each request uses a different IP. + +```bash +curl http://httpbin.org/ip --proxy http://127.0.0.1:3128 +# {"origin": "82.196.7.200"} -### Format +curl http://httpbin.org/ip --proxy http://127.0.0.1:3128 +# {"origin": "89.187.161.56"} ``` -openvpn/{name} -openvpn/{name}/{name2}.ovpn -openvpn/{name}/secret + +**Best for**: +- One-time requests +- High-volume scraping +- Load distribution + +### Mode 2: Static Proxy (Ports 30000+) + +Direct connection to individual proxy containers. Same IP for all requests to that port. + +```bash +curl http://httpbin.org/ip --proxy http://127.0.0.1:30000 +# {"origin": "82.196.7.200"} + +curl http://httpbin.org/ip --proxy http://127.0.0.1:30000 +# {"origin": "82.196.7.200"} # Same IP ``` -### 2. Generate docker-compose.yml +**Best for**: +- Browser automation (Selenium, Puppeteer, Playwright) +- Session-based scraping +- Login flows + +## Architecture + ``` -git clone https://github.com/39ff/docker-rotating-proxy -cd docker-rotating-proxy && cd setup -docker run --rm -it -v "$(pwd):/app" composer install +Client Request + ↓ +Squid (localhost:3128) - Rotating proxy mode + ├→ Direct HTTP Proxies (no container) + ├→ Gost Containers → SOCKS5/HTTP/HTTPS Proxies + └→ Gluetun Containers → OpenVPN → Internet + +OR + +Client Request → Gost/Gluetun Port (30000+) - Static IP mode +``` + +## OpenVPN Support + +To use VPN connections as proxies: + +### 1. Create OpenVPN Config + +``` +openvpn/{name}/ + ├── {config}.ovpn + └── secret # Optional: username on line 1, password on line 2 +``` + +**Example**: +``` +openvpn/nordvpn-us/ + ├── us123.nordvpn.com.ovpn + └── secret +``` + +### 2. Generate Config + +The generator will automatically: +- Detect all `.ovpn` files in `openvpn/` subdirectories +- Create Gluetun containers for each VPN +- Resolve hostnames to IPs (prevents DNS leaks) +- Expose HTTP proxy on ports 30000+ + +## Advanced Configuration + +### Custom Settings + +Copy and edit the config file: + +```bash +cp setup/config.php.example setup/config.php +``` + +**Available options**: + +```php + 30000, // Starting port for Gost + 'start_shadowsocks_port' => 50000, // Starting port for Shadowsocks + 'gluetun_http_port' => 8888, // HTTP port on Gluetun + 'squid_default_options' => '...', // Squid cache_peer options + 'enable_web_auth' => false, // Enable web-based user management +]; +``` + +### Web-Based User Management (Optional) + +This project integrates with [squid-db-auth-web](https://github.com/39ff/squid-db-auth-web) and [squid-db-auth-ip](https://github.com/39ff/squid-db-auth-ip) to provide a web UI for managing proxy users and authentication. + +**Features**: +- Web UI for user management +- Username/password authentication +- IP-based authentication +- Database-backed user credentials +- Role-based access control + +**Setup**: + +1. **Enable in configuration**: + +```bash +cp setup/config.php.example setup/config.php +``` + +Edit `setup/config.php`: + +```php + true, + + 'web_auth' => [ + 'web_port' => 8080, // Web UI port + 'db_name' => 'squidmin', + 'db_user' => 'squidmin', + 'db_password' => 'your_secure_password', // Change this! + 'db_root_password' => 'your_root_password', // Change this! + 'app_url' => 'http://localhost:8080', + ], +]; +``` + +2. **Generate configuration**: + +```bash +cd setup +php generate.php +``` + +This will automatically: +- Add MySQL database service +- Add Redis cache service +- Add Laravel application (squid-db-auth-web) +- Add Nginx web server +- Download authentication scripts +- Configure Squid to use database authentication + +3. **Start services**: + +```bash cd .. -# If you don't want to set up OpenVPN, please remove it. -rm -rf ./openvpn/* -docker run --rm -it -v "$(pwd):/app/" php:7.4-cli php /app/setup/generate.php -cat docker-compose.yml docker-compose up -d -curl https://httpbin.org/ip --proxy http://127.0.0.1:3128 -``` - -### How to it works? -![pattern1](https://user-images.githubusercontent.com/7544687/97991581-fdc2f380-1e24-11eb-99f3-df9885d627a2.png) - -- Sometimes you may need the same IP address for a series of steps. -To deal with this problem, we have built a new relay server via gost. - -- Most open proxies will be unavailable in a few days. -Therefore, it is useless to build a server for every open proxy, so we use squid's cache_peer to rotate a large number of open proxies. - -### proxyList.txt Example1 - -``` -127.0.0.1:1080:socks5:yourUsername:yourPassword -127.0.0.1:44129:httpsquid:mysquidproxy:mysquidpassword -127.0.0.1:29128:httpsquid:rotatingserviceUsername:password -169.254.0.1:1080:socks5:paidsocksUsername:paidsocksPassword -127.0.0.1:80 -172.31.22.222:8080 -``` - -## proxyList.txt Example2 -Here are some practical examples. - -using NordVPN,TorGuard,Luminati - -``` -89.187.161.86:80:httpsquid:yourNordVPNEmail@example.com:NordVPNPassword -173.254.222.146:1080:socks5:yourTorGuardUsername:Password -zproxy.lum-superproxy.io:22225:httpsquid:yourLuminatiUsername:Password -``` - - - -## Generated docker-compose.yml example -``` -version: '3.4' -services: - squid: - ports: - - '3128:3128' - image: 'b4tman/squid:5.8' - volumes: - - './config:/etc/squid/conf.d:ro' - container_name: dockersquid_rotate - environment: - - SQUID_CONFIG_FILE=/etc/squid/conf.d/squid.conf - extra_hosts: - - 'host.docker.internal:host-gateway' - healthcheck: - test: [CMD-SHELL, 'export https_proxy=127.0.0.1:3128 && export http_proxy=127.0.0.1:3128 && wget -q -Y on -O - https://checkip.amazonaws.com || exit 1'] - retries: 5 - timeout: 10s - start_period: 10s - interval: 300s - proxy1: - ports: - - '30000:30000' - image: 'ginuerzh/gost:latest' - container_name: dockergost_1 - command: '-L=:30000 -F=socks5://vpn:unlimited@82.196.7.200:2434' - vpn2: - ports: - - '30001:8888/tcp' - - '50000:8388' - image: qmcgaw/gluetun - container_name: dockervpn_2 - devices: - - '/dev/net/tun:/dev/net/tun' - cap_add: - - NET_ADMIN - volumes: - - './openvpn/hk-hkg.prod.surfshark.comsurfshark_openvpn_tcp.ovpn:/gluetun' - environment: - - VPN_SERVICE_PROVIDER=custom - - VPN_TYPE=openvpn - - OPENVPN_CUSTOM_CONFIG=/gluetun/vpn.ovpn - - HTTPPROXY=on - - HTTPPROXY_USER= - - HTTPPROXY_PASSWORD= - - HTTPPROXY_STEALTH=on - - OPENVPN_USER=xxxxx - - OPENVPN_PASSWORD=yyyyy - vpn3: - ports: - - '30002:8888/tcp' - - '50001:8388' - image: qmcgaw/gluetun - container_name: dockervpn_3 - devices: - - '/dev/net/tun:/dev/net/tun' - cap_add: - - NET_ADMIN - volumes: - - './openvpn/jp454.nordvpn.com.tcp443.ovpn:/gluetun' - environment: - - VPN_SERVICE_PROVIDER=custom - - VPN_TYPE=openvpn - - OPENVPN_CUSTOM_CONFIG=/gluetun/vpn.ovpn - - HTTPPROXY=on - - HTTPPROXY_USER= - - HTTPPROXY_PASSWORD= - - HTTPPROXY_STEALTH=on - - OPENVPN_USER=xxxxx - - OPENVPN_PASSWORD=yyyyy -``` - -## Now try it out -``` -port 3128 is rotation port. -Recommended for one-time requests that do not require browser rendering, such as curl - -sh-4.2# curl https://httpbin.org/ip --proxy https://127.0.0.1:3128 -{ - "origin": "82.196.7.200" -} -sh-4.2# curl https://httpbin.org/ip --proxy https://127.0.0.1:3128 -{ - "origin": "89.187.161.56" -} -sh-4.2# curl https://httpbin.org/ip --proxy https://127.0.0.1:3128 -{ - "origin": "84.17.37.159" -} -sh-4.2# curl https://httpbin.org/ip --proxy https://127.0.0.1:3128 -{ - "origin": "81.171.85.49" -} -sh-4.2# - -and.. try static ip gateway -Recommended in selenium, puppeteer and playwright - -# curl httpbin.org/ip --proxy http://127.0.0.1:30000 -{ - "origin": "82.196.7.200" -} -# curl httpbin.org/ip --proxy http://127.0.0.1:30000 -{ - "origin": "82.196.7.200" -} - -# curl httpbin.org/ip --proxy http://127.0.0.1:30001 -{ - "origin": "84.17.37.159" -} -# curl httpbin.org/ip --proxy http://127.0.0.1:30001 -{ - "origin": "84.17.37.159" -} -``` - - -## Warning -By default, ports can be used without authentication. -Some VPSs that are directly exposed globally may require appropriate modifications to the docker-compose. - - -## Example of using a large number of public proxies with real-time updates -see [public_proxy_cron.sh](public_proxy_cron.sh) -``` -0 * * * * /your_sh_path_here/public_proxy_cron.sh +``` + +Wait for services to initialize (about 30 seconds). + +4. **Initialize database** (first time only): + +```bash +# Run database migrations +docker-compose exec app php artisan migrate + +# Create admin user +docker-compose exec app php artisan db:seed --class=CreateAdministratorSeeder +``` + +5. **Access Web UI**: + +Open http://localhost:8080 in your browser. + +**Default admin credentials**: Check the seeder output or application documentation. + +**Using authenticated proxy**: + +```bash +# With username and password +curl http://httpbin.org/ip --proxy http://username:password@127.0.0.1:3128 + +# Test authentication +curl -v http://httpbin.org/ip --proxy http://127.0.0.1:3128 +# Should return 407 Proxy Authentication Required +``` + +**Managing users**: +- Add/remove users via web UI +- Set user quotas and bandwidth limits +- View usage statistics +- IP whitelist management + +**Architecture with authentication**: + +``` + Web Browser + ↓ + Web UI (localhost:8080) + ↙ ↘ + Laravel App MySQL DB + ↓ +Client → Squid (auth) → Database Check → Upstream Proxies + (port 3128) +``` + +### Public Proxy Auto-Update + +Use the included script to automatically fetch and update free proxies: + +```bash +# Edit crontab +crontab -e + +# Add line (runs hourly) +0 * * * * /path/to/public_proxy_cron.sh +``` + +See [public_proxy_cron.sh](public_proxy_cron.sh) for details. + +## Real-World Examples + +### NordVPN + +``` +# format: host:port:scheme:user:pass +89.187.161.86:80:httpsquid:your-email@example.com:your-password +``` + +### TorGuard + +``` +173.254.222.146:1080:socks5:your-username:your-password +``` + +### Luminati/Bright Data + +``` +zproxy.lum-superproxy.io:22225:httpsquid:your-username:your-password +``` + +### Free Proxy Lists + +``` +# Simple IP:Port format +192.168.1.100:8080 +172.31.22.222:3128 +``` + +Sources: [clarketm/proxy-list](https://github.com/clarketm/proxy-list) + +## Troubleshooting + +### Check Container Status + +```bash +docker-compose ps +``` + +### View Logs + +```bash +# All containers +docker-compose logs + +# Specific container +docker-compose logs squid +docker-compose logs proxy1 +``` + +### Test Individual Proxy + +```bash +# Test first Gost proxy +curl http://httpbin.org/ip --proxy http://127.0.0.1:30000 +``` + +### Regenerate Configuration + +```bash +cd setup +php generate.php +cd .. +docker-compose up -d +``` + +## Security Warning + +By default, proxies are accessible without authentication. If exposing to the internet: + +1. **Configure firewall rules** to restrict access +2. **Use allowed_ip.txt** to whitelist IPs +3. **Consider adding authentication** (see TODO) + +## Performance Tips + +- **Use `httpsquid` scheme** for HTTP proxies (no container overhead) +- **Remove unused OpenVPN configs** to reduce containers +- **Increase ulimits** if handling many connections +- **Monitor with** `docker stats` to check resource usage + +## File Structure + +``` +. +├── setup/ +│ ├── generate.php # Main generator script +│ ├── config.php.example # Configuration template +│ └── composer.json # PHP dependencies +├── template/ +│ ├── docker-compose.yml # Docker compose template +│ ├── squid.conf # Squid config template +│ └── allowed_ip.txt # IP whitelist template +├── config/ # Generated configs (created by script) +├── openvpn/ # OpenVPN configurations +├── proxyList.txt # Your proxy list +└── docker-compose.yml # Generated compose file + ``` ## TODO -- [ ] Username/Password Auth for Enterprise \ No newline at end of file + +- [x] Add username/password authentication for proxy access (via squid-db-auth-web) +- [x] Web UI for proxy management (via squid-db-auth-web) +- [ ] Health check and auto-removal of dead proxies +- [ ] Proxy performance metrics +- [ ] API endpoints for automation + +## Contributing + +Issues and pull requests are welcome at: https://github.com/39ff/docker-rotating-proxy + +## License + +See repository for license information. diff --git a/proxyList.txt b/proxyList.txt index 75d69c0..9ca4976 100644 --- a/proxyList.txt +++ b/proxyList.txt @@ -1,3 +1,5 @@ -159.89.206.161:2434:socks5:vpn:unlimited -142.93.68.63:2434:socks5:vpn:unlimited -82.196.7.200:2434:socks5:vpn:unlimited \ No newline at end of file +# Proxy list - you can customize the format below +# format: host,port,scheme,user,pass +159.89.206.161,2434,socks5,vpn,unlimited +142.93.68.63,2434,socks5,vpn,unlimited +82.196.7.200,2434,socks5,vpn,unlimited \ No newline at end of file diff --git a/proxyList.txt.example b/proxyList.txt.example new file mode 100644 index 0000000..1edc928 --- /dev/null +++ b/proxyList.txt.example @@ -0,0 +1,30 @@ +# Proxy List Configuration +# +# Format Options: +# 1. Use header line to define custom column order and delimiter: +# # format: host,port,scheme,user,pass +# # format: scheme:host:port:user:pass +# # format: port|host|scheme|user|pass +# +# 2. Without header (default): host:port:scheme:user:pass +# +# Supported delimiters: : (colon), , (comma), | (pipe), tab, space +# Supported schemes: socks5, http, https, httpsquid +# Lines starting with # are treated as comments + +# Example 1: Custom format with comma delimiter +# format: host,port,scheme,user,pass +159.89.206.161,2434,socks5,vpn,unlimited +142.93.68.63,2434,socks5,vpn,unlimited +82.196.7.200,2434,socks5,vpn,unlimited + +# Example 2: Simple IP:Port (no scheme = direct HTTP proxy) +# 192.168.1.100:8080 +# 10.0.0.50:3128 + +# Example 3: HTTP proxy with authentication +# proxy.example.com:8080:httpsquid:username:password + +# Example 4: Different order (scheme first) +# format: scheme,host,port,user,pass +# socks5,proxy.example.com,1080,user,pass diff --git a/setup/config.php.example b/setup/config.php.example new file mode 100644 index 0000000..0fcef72 --- /dev/null +++ b/setup/config.php.example @@ -0,0 +1,51 @@ + 30000, + + // Starting port for Shadowsocks (VPN) + 'start_shadowsocks_port' => 50000, + + // HTTP proxy port on Gluetun containers + 'gluetun_http_port' => 8888, + + // Default options for Squid cache_peer entries + 'squid_default_options' => 'no-digest no-netdb-exchange connect-fail-limit=2 connect-timeout=8 round-robin no-query allow-miss proxy-only', + + // Web-based authentication system + // Requires: https://github.com/39ff/squid-db-auth-web + // https://github.com/39ff/squid-db-auth-ip + 'enable_web_auth' => false, + + 'web_auth' => [ + // Web UI port (default: 8080) + 'web_port' => 8080, + + // MySQL database configuration + 'db_port' => 3306, + 'db_name' => 'squidmin', + 'db_user' => 'squidmin', + 'db_password' => 'squidmin_password', + 'db_root_password' => 'root_password', + + // Laravel app configuration + 'app_key' => '', // Leave empty to auto-generate + 'app_debug' => 'true', + 'app_url' => 'http://localhost:8080', + + // Redis configuration (used for sessions/cache) + 'redis_port' => 6379, + + // Path to squid-db-auth-web source code + // If empty, will use Docker image (recommended) + 'web_source_path' => '', + + // squid-db-auth-ip script path (will be copied to squid container) + 'auth_ip_script_url' => 'https://raw.githubusercontent.com/39ff/squid-db-auth-ip/main/basic_db_ip_auth.php', + ], +]; diff --git a/setup/generate.php b/setup/generate.php index 95a9885..6d84292 100644 --- a/setup/generate.php +++ b/setup/generate.php @@ -1,150 +1,552 @@ #!/usr/bin/env php 30000, + 'start_shadowsocks_port' => 50000, + 'gluetun_http_port' => 8888, + 'squid_default_options' => 'no-digest no-netdb-exchange connect-fail-limit=2 connect-timeout=8 round-robin no-query allow-miss proxy-only', +]; + +// Load user config if exists +if (file_exists(__DIR__.'/config.php')) { + $userConfig = require __DIR__.'/config.php'; + $config = array_merge($config, $userConfig); +} + +// Initialize +copy(__DIR__.'/../template/squid.conf', __DIR__.'/squid.conf'); +$dockerCompose = Yaml::parseFile(__DIR__.'/../template/docker-compose.yml'); + +// Process proxy list +$proxies = parseProxyList(__DIR__.'/../proxyList.txt'); +$state = [ + 'counter' => 1, + 'port' => $config['start_port'], + 'shadowsocks_port' => $config['start_shadowsocks_port'], +]; + +foreach ($proxies as $proxy) { + processProxy($proxy, $dockerCompose, $state, $config); +} + +// Process OpenVPN configurations +processOpenVPN($dockerCompose, $state, $config); + +// Add web authentication system if enabled +if (!empty($config['enable_web_auth'])) { + addWebAuthServices($dockerCompose, $config); +} + +// Write output files +file_put_contents(__DIR__.'/../docker-compose.yml', Yaml::dump($dockerCompose, 4, 4)); +rename(__DIR__.'/squid.conf', __DIR__.'/../config/squid.conf'); +copy(__DIR__.'/../template/allowed_ip.txt', __DIR__.'/../config/allowed_ip.txt'); + +echo "✓ Generated docker-compose.yml\n"; +echo "✓ Generated config/squid.conf\n"; +echo "✓ Processed " . ($state['counter'] - 1) . " proxy entries\n"; + +if (!empty($config['enable_web_auth'])) { + echo "✓ Web authentication enabled (http://localhost:" . $config['web_auth']['web_port'] . ")\n"; +} + +/** + * Parse proxy list file with flexible format support + * + * Supports header line to define column order: + * # format: host,port,scheme,user,pass + * or + * # format: scheme,host,port,user,pass + * + * Default format (no header): host:port:scheme:user:pass + */ +function parseProxyList($filepath) { + if (!file_exists($filepath)) { + echo "Warning: proxyList.txt not found\n"; + return []; + } + + $lines = file($filepath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + $proxies = []; + $format = ['host', 'port', 'scheme', 'user', 'pass']; // Default format + $delimiter = ':'; // Default delimiter + + foreach ($lines as $lineNum => $line) { + $line = trim($line); + + // Skip comments (but parse format header) + if (strpos($line, '#') === 0) { + // Check for format header + if (preg_match('/^#\s*format:\s*(.+)$/i', $line, $matches)) { + $formatStr = trim($matches[1]); + + // Detect delimiter (comma, colon, pipe, tab, space) + if (strpos($formatStr, ',') !== false) { + $delimiter = ','; + } elseif (strpos($formatStr, '|') !== false) { + $delimiter = '|'; + } elseif (strpos($formatStr, "\t") !== false) { + $delimiter = "\t"; + } elseif (strpos($formatStr, ' ') !== false) { + $delimiter = ' '; + } else { + $delimiter = ':'; + } + + $format = array_map('trim', explode($delimiter, $formatStr)); + echo "Using custom format: " . implode($delimiter, $format) . "\n"; + } + continue; } - }else{ - //other proxy type ex:socks - if ($proxyInfo['user'] && $proxyInfo['pass']) { - $cred = vsprintf('%s:%s@', array_map('urlencode', [$proxyInfo['user'], $proxyInfo['pass']])); + + // Skip empty lines + if (empty($line)) { + continue; } - $to['services']['proxy' . $i] = [ - 'ports' => [ - $port . ':' . $port, - ], + + // Parse proxy entry + $parts = array_map('trim', explode($delimiter, $line, count($format))); + $parts = array_pad($parts, count($format), ''); // Pad with empty strings + + $proxy = array_combine($format, $parts); + + // Validate required fields + if (empty($proxy['host']) || empty($proxy['port'])) { + echo "Warning: Skipping invalid proxy on line " . ($lineNum + 1) . ": $line\n"; + continue; + } + + // Normalize scheme + if (!empty($proxy['scheme'])) { + $proxy['scheme'] = strtolower($proxy['scheme']); + } + + $proxies[] = $proxy; + } + + return $proxies; +} + +/** + * Process a single proxy entry + */ +function processProxy($proxy, &$dockerCompose, &$state, $config) { + $squidConf = []; + $serviceName = 'proxy' . $state['counter']; + + // Build squid cache_peer template + $squidTemplate = sprintf( + 'cache_peer %%s parent %%d 0 %s name=%%s', + $config['squid_default_options'] + ); + + if (empty($proxy['scheme'])) { + // Direct HTTP proxy (no authentication, no container needed) + $squidConf[] = sprintf($squidTemplate, $proxy['host'], $proxy['port'], 'public' . $state['counter']); + } + elseif ($proxy['scheme'] === 'httpsquid') { + // HTTP proxy with potential authentication + $squidConf[] = sprintf($squidTemplate, $proxy['host'], $proxy['port'], 'private' . $state['counter']); + + if (!empty($proxy['user']) && !empty($proxy['pass'])) { + $squidConf[] = sprintf( + 'login=%s:%s', + urlencode($proxy['user']), + urlencode($proxy['pass']) + ); + } + } + else { + // SOCKS5/HTTP/HTTPS proxy - needs Gost container + $containerName = 'dockergost_' . $state['counter']; + $port = $state['port']; + + // Build credentials + $cred = ''; + if (!empty($proxy['user']) && !empty($proxy['pass'])) { + $cred = sprintf( + '%s:%s@', + urlencode($proxy['user']), + urlencode($proxy['pass']) + ); + } + + // Create Gost service + $dockerCompose['services'][$serviceName] = [ 'image' => 'ginuerzh/gost:latest', - 'container_name'=>'dockergost_'.$i, - 'command' => sprintf('-L=:%d -F=%s://%s%s:%d', $port, $proxyInfo['scheme'], $cred, $proxyInfo['host'], $proxyInfo['port']) + 'container_name' => $containerName, + 'restart' => 'unless-stopped', + 'ports' => [$port . ':' . $port], + 'command' => sprintf( + '-L=:%d -F=%s://%s%s:%d', + $port, + $proxy['scheme'], + $cred, + $proxy['host'], + $proxy['port'] + ), ]; - $squid_conf[] = sprintf($squid_default, 'dockergost_'.$i, $port, 'gost'.$i); - $port++; + + // Add to squid config + $squidConf[] = sprintf($squidTemplate, $containerName, $port, 'gost' . $state['counter']); + + $state['port']++; } - if($squid_conf){ - file_put_contents(__DIR__.'/squid.conf', PHP_EOL . implode(' ', $squid_conf), FILE_APPEND); + + // Write squid configuration + if (!empty($squidConf)) { + file_put_contents( + __DIR__.'/squid.conf', + PHP_EOL . implode(' ', $squidConf), + FILE_APPEND + ); } - $i++; + + $state['counter']++; } -//openvpn support -if(file_exists(__DIR__.'/../openvpn')){ - foreach(glob(__DIR__.'/../openvpn/*') as $fileOrDir){ - if(!is_dir($fileOrDir)){ - continue; - } - $config_ovpn = glob(__DIR__.'/../openvpn/'.basename($fileOrDir).'/*.ovpn'); - if(empty($config_ovpn[0])){ - throw new \RuntimeException("OpenVPN Configuration File Not Found:".$fileOrDir); - } +/** + * Process OpenVPN configurations + */ +function processOpenVPN(&$dockerCompose, &$state, $config) { + $openvpnDir = __DIR__.'/../openvpn'; - $config_secret = glob(__DIR__.'/../openvpn/'.basename($fileOrDir).'/secret'); - $config_ovpn = realpath($config_ovpn[0]); + if (!file_exists($openvpnDir)) { + return; + } - //If a hostname is present in the Config, convert it to an IP address. (To prevent IP address leaks) - $config_lists = file($config_ovpn,FILE_IGNORE_NEW_LINES|FILE_SKIP_EMPTY_LINES); - foreach($config_lists as $key=>$config_list){ - if(preg_match("/^remote\s+([\w.-]+)(?:\s+(\d+))?$/",$config_list,$matches)){ - $remote = 'remote '.gethostbyname($matches[1]); - if(isset($matches[2])){ - $remote.= ' '.$matches[2]; - } - $config_lists[$key] = $remote; + $squidTemplate = sprintf( + 'cache_peer %%s parent %%d 0 %s name=%%s', + $config['squid_default_options'] + ); - file_put_contents($config_ovpn,implode(PHP_EOL,$config_lists)); - break; - } + foreach (glob($openvpnDir . '/*', GLOB_ONLYDIR) as $configDir) { + $ovpnFiles = glob($configDir . '/*.ovpn'); + + if (empty($ovpnFiles[0])) { + echo "Warning: No .ovpn file found in " . basename($configDir) . "\n"; + continue; } - $credentials = null; + $ovpnFile = realpath($ovpnFiles[0]); + $secretFile = $configDir . '/secret'; + + // Resolve hostname to IP (prevent DNS leaks) + resolveOpenVPNHostname($ovpnFile); + + // Prepare environment variables $env = [ 'VPN_SERVICE_PROVIDER=custom', 'VPN_TYPE=openvpn', - 'OPENVPN_CUSTOM_CONFIG=/gluetun/'.basename($config_ovpn), + 'OPENVPN_CUSTOM_CONFIG=/gluetun/' . basename($ovpnFile), 'HTTPPROXY=on', 'HTTPPROXY_USER=', 'HTTPPROXY_PASSWORD=', 'HTTPPROXY_STEALTH=on', ]; - if(!empty($config_secret[0])) { - $credentials = file($config_secret[0],FILE_IGNORE_NEW_LINES|FILE_SKIP_EMPTY_LINES); - $env = array_merge($env,[ - 'OPENVPN_USER='.$credentials[0], - 'OPENVPN_PASSWORD='.$credentials[1], - ]); + // Add credentials if secret file exists + if (file_exists($secretFile)) { + $credentials = file($secretFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + if (count($credentials) >= 2) { + $env[] = 'OPENVPN_USER=' . $credentials[0]; + $env[] = 'OPENVPN_PASSWORD=' . $credentials[1]; + } } - $to['services']['vpn' . $i] = [ - 'ports' => [ - $port . ':'.$gluetun_http_port.'/tcp', - $port_shadow_socks . ':8388', - ], + // Create VPN service + $serviceName = 'vpn' . $state['counter']; + $containerName = 'dockervpn_' . $state['counter']; + + $dockerCompose['services'][$serviceName] = [ 'image' => 'qmcgaw/gluetun', - 'container_name'=>'dockervpn_'.$i, - 'devices'=>[ - '/dev/net/tun:/dev/net/tun' - ], - 'cap_add'=>[ - 'NET_ADMIN' - ], - 'volumes'=>[ - './openvpn/'.basename($fileOrDir).':/gluetun' + 'container_name' => $containerName, + 'restart' => 'unless-stopped', + 'devices' => ['/dev/net/tun:/dev/net/tun'], + 'cap_add' => ['NET_ADMIN'], + 'ports' => [ + $state['port'] . ':' . $config['gluetun_http_port'] . '/tcp', + $state['shadowsocks_port'] . ':8388', ], - 'environment'=>$env + 'volumes' => ['./openvpn/' . basename($configDir) . ':/gluetun'], + 'environment' => $env, ]; - file_put_contents(__DIR__.'/squid.conf', PHP_EOL . sprintf($squid_default, 'dockervpn_'.$i, $gluetun_http_port, 'vpn'.$i), FILE_APPEND); - $i++; - $port++; - $port_shadow_socks++; + // Add to squid config + file_put_contents( + __DIR__.'/squid.conf', + PHP_EOL . sprintf($squidTemplate, $containerName, $config['gluetun_http_port'], 'vpn' . $state['counter']), + FILE_APPEND + ); + + $state['counter']++; + $state['port']++; + $state['shadowsocks_port']++; + } +} + +/** + * Resolve OpenVPN hostname to IP address to prevent DNS leaks + */ +function resolveOpenVPNHostname($ovpnFile) { + $lines = file($ovpnFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + $modified = false; + + foreach ($lines as $key => $line) { + if (preg_match('/^remote\s+([\w.-]+)(?:\s+(\d+))?$/', $line, $matches)) { + $hostname = $matches[1]; + $ip = gethostbyname($hostname); + + // Only replace if DNS resolution succeeded (IP != hostname) + if ($ip !== $hostname) { + $remote = 'remote ' . $ip; + if (isset($matches[2])) { + $remote .= ' ' . $matches[2]; + } + $lines[$key] = $remote; + $modified = true; + echo "Resolved VPN hostname: $hostname -> $ip\n"; + } + break; + } + } + + if ($modified) { + file_put_contents($ovpnFile, implode(PHP_EOL, $lines)); + } +} + +/** + * Add web authentication services to docker-compose + * Integrates squid-db-auth-web and squid-db-auth-ip for user management + */ +function addWebAuthServices(&$dockerCompose, $config) { + $webAuth = $config['web_auth']; + + // Generate Laravel APP_KEY if not provided + $appKey = $webAuth['app_key']; + if (empty($appKey)) { + $appKey = 'base64:' . base64_encode(random_bytes(32)); + } + + // Add MySQL service + $dockerCompose['services']['db'] = [ + 'image' => 'mysql:8.0', + 'container_name' => 'dockersquid_mysql', + 'restart' => 'unless-stopped', + 'ports' => [$webAuth['db_port'] . ':3306'], + 'environment' => [ + 'MYSQL_ROOT_PASSWORD=' . $webAuth['db_root_password'], + 'MYSQL_DATABASE=' . $webAuth['db_name'], + 'MYSQL_USER=' . $webAuth['db_user'], + 'MYSQL_PASSWORD=' . $webAuth['db_password'], + 'TZ=UTC', + ], + 'volumes' => ['db-store:/var/lib/mysql'], + 'command' => '--default-authentication-plugin=mysql_native_password', + ]; + + // Add Redis service + $dockerCompose['services']['redis'] = [ + 'image' => 'redis:6.2-alpine', + 'container_name' => 'dockersquid_redis', + 'restart' => 'unless-stopped', + 'ports' => [$webAuth['redis_port'] . ':6379'], + 'volumes' => ['redis-store:/data'], + ]; + + // Add Laravel application service + $dockerCompose['services']['app'] = [ + 'image' => 'ghcr.io/39ff/squid-db-auth-web:latest', + 'container_name' => 'dockersquid_app', + 'restart' => 'unless-stopped', + 'depends_on' => ['db', 'redis'], + 'environment' => [ + 'APP_NAME=SquidUserManager', + 'APP_ENV=production', + 'APP_KEY=' . $appKey, + 'APP_DEBUG=' . $webAuth['app_debug'], + 'APP_URL=' . $webAuth['app_url'], + 'DB_CONNECTION=mysql', + 'DB_HOST=db', + 'DB_PORT=3306', + 'DB_DATABASE=' . $webAuth['db_name'], + 'DB_USERNAME=' . $webAuth['db_user'], + 'DB_PASSWORD=' . $webAuth['db_password'], + 'REDIS_HOST=redis', + 'REDIS_PASSWORD=null', + 'REDIS_PORT=6379', + 'CACHE_DRIVER=redis', + 'SESSION_DRIVER=redis', + 'QUEUE_CONNECTION=sync', + ], + ]; + + // If source path is provided, mount it + if (!empty($webAuth['web_source_path'])) { + $dockerCompose['services']['app']['volumes'] = [ + $webAuth['web_source_path'] . ':/app' + ]; + } + + // Add Nginx web server + $dockerCompose['services']['web'] = [ + 'image' => 'nginx:alpine', + 'container_name' => 'dockersquid_web', + 'restart' => 'unless-stopped', + 'ports' => [$webAuth['web_port'] . ':80'], + 'depends_on' => ['app'], + 'volumes' => [ + './config/nginx.conf:/etc/nginx/conf.d/default.conf:ro', + ], + ]; + + // Update Squid service to depend on db and include auth script + if (isset($dockerCompose['services']['squid'])) { + if (!isset($dockerCompose['services']['squid']['depends_on'])) { + $dockerCompose['services']['squid']['depends_on'] = []; + } + $dockerCompose['services']['squid']['depends_on'][] = 'db'; + + // Add volume for auth script + if (!isset($dockerCompose['services']['squid']['volumes'])) { + $dockerCompose['services']['squid']['volumes'] = []; + } + $dockerCompose['services']['squid']['volumes'][] = './config/basic_db_ip_auth.php:/etc/squid/basic_db_ip_auth.php:ro'; + } + + // Add volumes for persistence + if (!isset($dockerCompose['volumes'])) { + $dockerCompose['volumes'] = []; + } + $dockerCompose['volumes']['db-store'] = [ + 'driver' => 'local', + 'driver_opts' => ['type' => 'none', 'o' => 'bind', 'device' => './volumes/mysql'], + ]; + $dockerCompose['volumes']['redis-store'] = [ + 'driver' => 'local', + 'driver_opts' => ['type' => 'none', 'o' => 'bind', 'device' => './volumes/redis'], + ]; + + // Create volumes directory structure + @mkdir(__DIR__.'/../volumes', 0755, true); + @mkdir(__DIR__.'/../volumes/mysql', 0755, true); + @mkdir(__DIR__.'/../volumes/redis', 0755, true); + + // Download and save auth IP script + downloadAuthScript($webAuth['auth_ip_script_url'], __DIR__.'/../config/basic_db_ip_auth.php'); + // Generate nginx configuration + generateNginxConfig(__DIR__.'/../config/nginx.conf'); + + // Generate auth-enabled squid configuration + generateAuthSquidConfig(__DIR__.'/squid.conf', $webAuth); + + echo "Setting up web authentication services...\n"; + echo "- MySQL database\n"; + echo "- Redis cache\n"; + echo "- Laravel application\n"; + echo "- Nginx web server\n"; +} + +/** + * Download authentication script from GitHub + */ +function downloadAuthScript($url, $destination) { + echo "Downloading auth script from $url...\n"; + + $content = @file_get_contents($url); + if ($content === false) { + echo "Warning: Could not download auth script. Using placeholder.\n"; + $content = "