CentOS7 Squid + SquidGuard + blacklist

From Bitbull Wiki
Jump to navigation Jump to search

1 Links

2 Setup

2.1 Install Squid

yum -y install epel-release
yum -y install squid perl-Crypt-OpenSSL-X509 squidGuard httpd

2.2 Iptables

yum -y remove firewalld
yum -y install iptables iptables-utils iptables-services
vi /etc/sysconfig/iptables
*mangle
# prevent squid transparent ports from looping while tcp connect to host (nagios)
-A PREROUTING -p tcp -m tcp --dport 3128 -j DROP
-A PREROUTING -p tcp -m tcp --dport 3129 -j DROP
COMMIT

*nat
# internal traffic
# -A PREROUTING -m limit --limit 1/sec --limit-burst 7 -j LOG --log-prefix "[IPTABLES PREROUTING NAT "
-A PREROUTING -s 192.168.223.0/24 ! -d 192.168.223.0/24  -p tcp -m tcp --dport 21  -j DNAT --to-destination 192.168.223.60:3128
-A PREROUTING -s 192.168.223.0/24 ! -d 192.168.223.0/24  -p tcp -m tcp --dport 80  -j DNAT --to-destination 192.168.223.60:3128
-A PREROUTING -s 192.168.223.0/24 ! -d 192.168.223.0/24  -p tcp -m tcp --dport 443 -j DNAT --to-destination 192.168.223.60:3129
COMMIT

*filter
# -A INPUT  -m limit --limit 1/s --limit-burst 7   -j LOG --log-prefix "[IPTABLES INPUT "
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A INPUT -p icmp -j ACCEPT
-A INPUT -i lo -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 21 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 80 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 443 -j ACCEPT
-A INPUT -m udp -p udp --dport 161 -j ACCEPT
#http transparent
-A INPUT -p tcp -m state --state NEW -m tcp --dport 3128 -j ACCEPT
#https transparent
-A INPUT -p tcp -m state --state NEW -m tcp --dport 3129 -j ACCEPT
#classic proxy
-A INPUT -p tcp -m state --state NEW -m tcp --dport 8080 -j ACCEPT
-A INPUT -j REJECT --reject-with icmp-host-prohibited

# -A FORWARD -m limit --limit 1/sec --limit-burst 7 -j LOG --log-prefix "[IPTABLES FORWARD "
-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -p icmp -j ACCEPT
-A FORWARD -p tcp -m state --state NEW -m tcp --sport 22 -j ACCEPT
-A FORWARD -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
# -A FORWARD -p tcp -m state --state NEW -m tcp --dport 11371 -j ACCEPT #gpg server for apt
-A FORWARD -p udp -m udp --dport 1194 -j ACCEPT
-A FORWARD -j REJECT --reject-with icmp-host-prohibited
COMMIT
vi /etc/sysconfig/iptables-config
IPTABLES_MODULES="ip_nat_ftp ip_conntrack ip_conntrack_ftp"

systemctl restart iptables

2.3 Open System Limitations

echo "
# Increase file descriptor limits for Squid
squid               soft    nofile          65536
squid               hard    nofile          65536
" > /etc/security/limits.d/squid.conf
vi /etc/sysctl.conf
# Controls IP packet forwarding
net.ipv4.ip_forward = 1
# Controls source route verification
net.ipv4.conf.default.rp_filter = 0
# Do not accept source routing
net.ipv4.conf.default.accept_source_route = 0

# We don’t use IPv6, so no point in having it enabled really
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1

# Increase local port range to support more concurrent connections
# cat /proc/sys/net/ipv4/ip_local_port_range – defaults to 32768 – 61000
net.ipv4.ip_local_port_range = 1025 65535

# Increase limit of system-wide file descriptors
# cat /proc/sys/fs/file-max
fs.file-max = 65536

# Allow a greater number of half-opened TCP connections, mitigate “possible SYN flooding” warnings in the messages log
# cat /proc/sys/net/ipv4/tcp_max_syn_backlog – defaults to 1024
net.ipv4.tcp_max_syn_backlog = 2048

# Make sure syn cookies are enabled too
net.ipv4.tcp_syncookies = 1

# tune tcp
net.core.somaxconn = 12800
net.core.netdev_max_backlog = 100000
net.ipv4.tcp_max_syn_backlog = 204800

sysctl -p

2.4 Filter Update Script

vi /usr/local/sbin/update_url_filter.sh
#!/bin/bash
#DESC: update blacklist and adfilter

export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin

### UPDATE ALL FILTERS ################################################################################
wget -q -O - 'http://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml' > /etc/squid/ad_block.txt
wget -q -O - http://www.shallalist.de/Downloads/shallalist.tar.gz > /tmp/shallalist.tar.gz

### UPDATE BLACKLIST FROM SHALLA ######################################################################
cd /tmp/
test -d /var/squidGuard/blacklists || mkdir /var/squidGuard/blacklists
tar xfz /tmp/shallalist.tar.gz
rsync -a /tmp/BL/* /var/squidGuard/blacklists/

### UPDATE AD FILTERS #################################################################################
# add . at beginning of domain, to match subdomains, remove ips
cat /var/squidGuard/blacklists/adv/domains >> /etc/squid/ad_block.txt
sed -i '/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/! {s/^\.//g;s/^/./g}'  /etc/squid/ad_block.txt

rm -rf BL /tmp/shallalist.tar.gz
test -f /var/squidGuard/blacklists.tar.gz && rm -f /var/squidGuard/blacklists.tar.gz

find /var/squidGuard/blacklists/ -type f -name '*.db' -exec rm -f {} \;
squidGuard -C all
chown -R squid.squid /var/squidGuard
systemctl restart squid

chmod 700 /usr/local/sbin/update_url_filter.sh
bash -x /usr/local/sbin/update_url_filter.sh

2.5 Configure SquidGuard

cp /etc/squid/squidGuard.conf /etc/squid/squidGuard.conf.orig

sed -i 's|proxymaster\\@foo.bar|admin\\@bitbull.ch|g' /var/www/cgi-bin/squidGuard*cgi
vi /usr/local/sbin/gen_bl_conf.sh
echo '
### BASE DIRS ###
dbhome /var/squidGuard/blacklists
logdir /var/log/squidGuard

### BLACKLISTS FROM SHALLA ###
'
cd /var/squidGuard/blacklists/
find . -type f | grep db$ | xargs rm -f
find ./ -type d | cut -d/ -f2- | egrep -v '^$' | while read BL
do
   DOM=$(ls $BL/domains 2>/dev/null)
   URL=$(ls $BL/urls 2>/dev/null)
   echo $DOM $URL | egrep -q "/domains|/urls"
   if [ $? -eq 0 ]
   then
      echo "$BL" | grep -q whitelist || echo -n "!bl_$BL " >> /tmp/$(basename $0)-$$.tmp
      echo "#---------- $BL ----------"
      egrep -A7 "NAME:.*$BL$" global_usage | egrep 'DEFAULT_TYPE:|DESC DE:' | sed 's/^/#/g'
      echo "destination bl_$BL {"
      [ "x" = "x$DOM" ] || echo "          domainlist      $DOM"
      [ "x" = "x$URL" ] || echo "          urllist         $URL"
      echo "}"
      fi
done

echo "
### FORCE GOOGLE SAFE SEARCH ###
rewrite safesearch {
    s@(google\..*/search.*q=.*)@\1\&safe=active@i
    s@(google\..*/images.*q=.*)@\1\&safe=active@i
    s@(google\..*/groups.*q=.*)@\1\&safe=active@i
    s@(google\..*/news.*q=.*)@\1\&safe=active@i
    s@(bing\..*/search.*q=.*)@\1\&adlt=strict@i
    s@(bing\..*/videos.*q=.*)@\1\&adlt=strict@i
    s@(bing\..*/images.*q=.*)@\1\&adlt=strict@i
    s@(search.yahoo\..*/search.*p=.*)@\1\&vm=r@i
    s@(duckduckgo.com\..*/.*q=.*)@\1\&kp=1@i
}

### SET DEFAULT ACL ###
acl {
        default {
                rewrite safesearch
                pass bl_whitelist $(cat /tmp/$(basename $0)-$$.tmp | sed 's/! //')
                redirect http://$(hostname )/cgi-bin/squidGuard-simple-de.cgi?clientaddr=%a&clientname=%n&clientident=%i&clientgroup=%s&targetgroup=%t&url=%u
                }

}
"

rm -f /tmp/$(basename $0)-$$.tmp


bash /usr/local/sbin/gen_bl_conf.sh
  • edit squidGuard.conf according your needs
mkdir -p /var/squidGuard/blacklists/{blacklist,whitelist}
echo whitelist.com > /var/squidGuard/blacklists/whitelist/domains
echo blacklist.com > /var/squidGuard/blacklists/blacklist/domains

squidGuard -C all
chown -R squid.squid /var/squidGuard

2.6 Configure SSL Proxy

vi /etc/pki/tls/openssl.cnf
default_days 			= 3650
countryName_default		= CH
stateOrProvinceName_default	= St Gall
localityName_default		= Flawil
0.organizationName_default	= Bitbull
organizationalUnitName_default	= Unix Support
commonName_default		= proxy1.office.bitbull.ch
emailAddress_default		= support@bitbull.ch

rm -rf /etc/squid/ssl_cert
mkdir -p /etc/squid/ssl_cert
cd /etc/squid/ssl_cert
openssl genrsa -out squid.key 2048
openssl req -new -key squid.key -out squid.csr
openssl x509 -req -days 3650 -in squid.csr -signkey squid.key -out squid.crt
cat squid.key squid.crt > squid.pem #browser cert
openssl x509 -in squid.pem -outform DER -out squid.der #browser cert ff ie

rm -fr /var/lib/ssl_db
/usr/lib64/squid/ssl_crtd -c -s /var/lib/ssl_db
chown -R squid.squid /var/lib/ssl_db

2.7 Configure AD Blocker

cp -a /usr/share/squid/errors/de /etc/squid/pages
echo '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"><html><body>Werbung entfernt</body></html>' > /etc/squid/pages/ERR_NO_AD

echo ".stackexchange.com" > /etc/squid/ad_block_ignore.txt
echo ".taobao.com
.alicdn.com
.mmstat.com
.tbcdn.cn
.greencompute.org
.chartbeat.net
.googlesyndication.com
.googleadservices.com" > /etc/squid/ad_block_custom.txt

2.8 Configure Squid

cp /etc/squid/squid.conf /etc/squid/squid.conf.orig
vi /etc/squid/squid.conf
# Adapt to list your (internal) IP networks from where browsing should be allowed
acl localnet src 10.0.0.0/8	# RFC1918 possible internal network
acl localnet src 172.16.0.0/12	# RFC1918 possible internal network
acl localnet src 192.168.0.0/16	# RFC1918 possible internal network

acl SSL_ports port 443
acl Safe_ports port 80		# http
acl Safe_ports port 21		# ftp
acl Safe_ports port 443		# https
#acl Safe_ports port 1025-65535	# unregistered ports
acl CONNECT method CONNECT

# Deny requests to certain unsafe ports
http_access deny !Safe_ports

# Deny CONNECT to other than secure SSL ports
http_access deny CONNECT !SSL_ports

# Only allow cachemgr access from localhost
http_access allow localhost manager
http_access deny manager

# deny nasty things
http_access deny to_localhost

### ADFILTER CONFIG ###
error_directory /etc/squid/pages
acl ads dstdom_regex -i "/etc/squid/ad_block.txt"
acl myads dstdom_regex -i "/etc/squid/ad_block_custom.txt"
acl myads_ignore dstdom_regex -i "/etc/squid/ad_block_ignore.txt"
http_access allow myads_ignore
http_access deny  myads
http_access deny  ads
deny_info ERR_NO_AD ads
deny_info ERR_NO_AD myads

#   ### LDAP AUTH ###
#   auth_param basic program /usr/lib64/squid/basic_ldap_auth -v 3 -b "dc=bitbull,dc=ch" -f uid=%s ldap1.bitbull.ch
#   auth_param basic children 5
#   auth_param basic realm Web-Proxy
#   auth_param basic credentialsttl 1 minute
#   acl ldap-auth proxy_auth REQUIRED
#   http_access allow ldap-auth

# rule allowing access from your local networks
http_access allow localnet
http_access allow localhost

# And finally deny all other access to this proxy
http_access deny all

# Squid normally listens to port 3128
http_port  8080

# Uncomment and adjust the following to add a disk cache directory.
# cache_dir ufs /var/spool/squid 100 16 256

# Leave coredumps in the first cache dir
coredump_dir /var/spool/squid

# Add any of your own refresh_pattern entries above these.
refresh_pattern ^ftp:		1440	20%	10080
refresh_pattern ^gopher:	1440	0%	1440
refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
refresh_pattern .		0	20%	4320

### SSL CONFIG ###
# EXCLUDE SSL ACL
acl ssl-ignore-hosts dstdomain "/etc/squid/ssl-ignore-hosts.acl"
acl ssl-ignore-ips dst "/etc/squid/ssl-ignore-ips.acl"

### SSL CONFIG ###
http_port 3128 intercept
https_port 3129 intercept ssl-bump  generate-host-certificates=on dynamic_cert_mem_cache_size=4MB cert=/etc/squid/ssl_cert/squid.pem
ssl_bump none localhost
ssl_bump none ssl-ignore-hosts
ssl_bump none ssl-ignore-ips
sslproxy_cert_error allow all
# sslproxy_flags DONT_VERIFY_PEER
ssl_bump server-first all


#   ### WCCP CONFIG ###
#   # additional port for transparent proxy
#   http_port 3127 transparent
#   # WCCP Router IP
#   wccp2_router 10.0.0.1
#   wccp2_router 10.0.8.1
#   # forwarding 1=gre 2=l2
#   wccp2_forwarding_method 1
#   # GRE return method gre|l2
#   wccp2_return_method 1
#   # Assignment method hash|mask
#   wccp2_assignment_method hash
#   # standard web cache, no auth
#   wccp2_service standard 0

### vi /etc/sysconfig/network-scripts/ifcfg-tun0
# ----------
# DEVICE=tun0
# BOOTPROTO=none
# ONBOOT=yes
# TYPE=GRE
# PEER_OUTER_IPADDR=141.136.108.122
# PEER_INNER_IPADDR=192.168.77.254
# MY_INNER_IPADDR=192.168.77.253
# ----------

### SERVER SPECIFIC CONF ###
visible_hostname proxy1.office.bitbull.ch
dns_nameservers 192.168.223.50
append_domain .office.bitbull.ch
cache deny localnet
ipcache_size 10240
negative_dns_ttl 5 minutes
forwarded_for delete
cache_mgr support@bitbull.ch
max_filedesc 16384
cache_mem 512 MB

# MODIFICATION FOR SQUIDGUARD
url_rewrite_program /usr/bin/squidGuard

echo "proxy1.office.bitbull.ch OK" > /var/www/html/index.html
  • Configure SSL bumping exclude rules for "broken but trusted" and "high secure (banking, ...)"
note, that the most important part is the ip file, because traffic, which is not inspected, does not see the hostheader
SNI is not possible with squid 3.3, and in most cases for "broken but trusted" servers, it is anyway not available.


If you do not use IP for SSL splicing, you will get often this message in squid log:
cache.log -> fwdNegotiateSSL: Error negotiating SSL connection on FD 15: error:00000000:lib(0):func(0):reason(0) (5/0/0)
access.log -> 192.168.223.58 TCP_MISS/200 0 CONNECT 13.16.33.124:443 - HIER_DIRECT/13.16.33.124 -
browser error message ->
The system returned:
    (71) Protocol error (TLS code: SQUID_ERR_SSL_HANDSHAKE)
    Handshake with SSL server failed: [No Error]



echo eppns3.eur.xerox.com > /etc/squid/ssl-ignore-hosts.acl
echo 13.16.33.124 > /etc/squid/ssl-ignore-ips.acl
  • You can verify splicing easily by by checking connection cert on https site when browsing
squid-ca -> site is bumped by squid (MITM, but normally this fails, that is the main reason to splice https sites)
original website ca -> site is spliced by squid (traffic is not modified)
systemctl enable squid httpd iptables
systemctl restart squid httpd iptables
Retrieved from "http://www.bitbull.ch/wiki/index.php?title=CentOS7_Squid_%2B_SquidGuard_%2B_blacklist&oldid=3391"