New prerelease 1 (on top of 3.429)

LATESTIS

 429
-0
+1

README

@@ -23,11 +23,12 @@ stalag13-utils-cache-nginx:
 
 stalag13-utils-cache-spoof:
  DNS spoofer needed by cache setups 
- /etc/bind/named.conf.views must be included in named.conf
- /etc/bind/named.conf.acl must be edited to match the cache host IP
- /etc/bind/db.cache must be edited to match the cache host IP
- /etc/bind/named.conf.cache-rebuild.sh can be edited and re-run if changed
- /etc/bind/named.conf.ads usage is optional
+ /etc/powerdns/recursor.conf must mention lua-dns-script=/etc/powerdns/redirect.lua
+ in /etc/powerdns/ :
+ redirect-cached-rebuild.sh must be edited and executed to build required redirect-cached.lua
+ redirect-ads-rebuil.pl must be executed (and probably set a weekly  cronjob) to build
+ required redirect-ads.lua
+ redirect-blacklisted.lua is a simple hand maintained domains blacklist. 
 
 stalag13-utils-cache-steam:
  Steam downloads cache setup

debian/changelog

-changelog.releases
\ No newline at end of file
+changelog.full
\ No newline at end of file

debian/changelog.full

+stalag13-utils (3.429+1-20170622) unstable; urgency=low
+ 
+  * Upstream prerelease
+ 
+ -- Mathieu Roy <klink@bender.stalag13.ici>  Thu, 22 Jun 2017 14:36:07 +0200
+ 
 stalag13-utils (3.429-20170621) unstable; urgency=low
  
   * Cosmetics. Workaround bind9 limit s writeable file '...' already in use... : by using in-view (not overly satisfying but works). Update devuan/debian cache DNS

debian/control

@@ -42,14 +42,15 @@ Description: nginx setup specific to caching servers
 
 Package: stalag13-utils-cache-spoof
 Architecture: all
-Depends: stalag13-utils, stalag13-utils-nginx, bind9
+Depends: stalag13-utils, stalag13-utils-nginx, pdns-recursor
 Conflicts: stalag13-utils (<<3.166), stalag13-utils-ahem (<<3.222)
 Description: DNS spoofer needed by cache setups 
- /etc/bind/named.conf.views must be included in named.conf
- /etc/bind/named.conf.acl must be edited to match the cache host IP
- /etc/bind/db.cache must be edited to match the cache host IP
- /etc/bind/named.conf.cache-rebuild.sh can be edited and re-run if changed
- /etc/bind/named.conf.ads usage is optional
+ /etc/powerdns/recursor.conf must mention lua-dns-script=/etc/powerdns/redirect.lua
+ in /etc/powerdns/ :
+ redirect-cached-rebuild.sh must be edited and executed to build required redirect-cached.lua
+ redirect-ads-rebuil.pl must be executed (and probably set a weekly  cronjob) to build
+ required redirect-ads.lua
+ redirect-blacklisted.lua is a simple hand maintained domains blacklist. 
  .
  Homepage: https://yeupou.wordpress.com/
  

debian/repack.pl

@@ -49,7 +49,7 @@ my %packages = (utils => ["/etc/bash_completion.d", "/etc/bashrc.d", "/etc/profi
 		"utils-cache-apt" => ["/etc/nginx/sites-available/cache-apt", "/etc/nginx/conf.d/cache-apt.conf", "/etc/cron.weekly/cache-apt"],
 		"utils-cache-steam" => ["/etc/nginx/sites-available/cache-steam", "/etc/nginx/conf.d/cache-steam.conf", "/etc/cron.daily/cache-steam"],
 		"utils-cache-nginx" => ["/etc/resolvconf/update-libc.d/nginx", "/etc/nginx/conf.d/resolver.conf"],
-		"utils-cache-spoof" => ["/etc/bind", "/usr/local/bin/update-bind-ads-block.pl", "/etc/dhcp/dhclient-exit-hooks.d/bind"],
+		"utils-cache-spoof" => ["/etc/powerdns"],
 		"utils-exim" => ["/etc/spamassassin", "/etc/exim4", "/usr/local/bin/memcached-exim.pl", "/etc/cron.hourly/exim-final_from", "/etc/cron.d/stalag13-spamslayer", "/usr/local/bin/stalag13-spamslayer-fixrights.sh", "/usr/local/bin/stalag13-spamslayer-learn.sh", "/usr/local/share/doc/dmarcts-report-parser/", "/usr/local/bin/dmarcts-report-parser.pl"],
 		"utils-munin" => ["/usr/share/munin","/usr/local/share/munin", "/etc/munin", "/etc/cron.d/munin-sync", "/usr/local/bin/munin-cron-plus.pl"],	
 		"utils-nginx" => ["/etc/nginx/conf.d/ssl.conf", "/etc/nginx/conf.d/discretion.conf", "/etc/nginx/cache_proxy_params", "/etc/nginx/allow_local", "/etc/nginx/sites-available/0blank"],

etc/bind/db.ads

-; File: null.zone
-; Last modified: 07-10-2005
-
-$TTL    86400   ; one day
-
-@       IN      SOA     localhost.   root.localhost. (
-                        2005071005       ; serial number YYYYMMDDNN
-                        28800   ; refresh  8 hours
-                        7200    ; retry    2 hours
-                        864000  ; expire  10 days
-                        86400 ) ; min ttl  1 day
-                NS      localhost.
-
-                A       127.0.0.1
-
-*               IN      A       127.0.0.1

etc/bind/db.cache

-; File: null.zone
-; Last modified: 05-10-2017
-
-$TTL    86400   ; one day
-
-@       IN      SOA     localhost.   root.localhost. (
-                        2005071005       ; serial number YYYYMMDDNN
-                        28800   ; refresh  8 hours
-                        7200    ; retry    2 hours
-                        864000  ; expire  10 days
-                        86400 ) ; min ttl  1 day
-                NS      localhost.
-
-; should point to the cache host IP
-                A       10.0.0.88
-*		A	10.0.0.88

etc/bind/named.conf.acl

-// defines loopback and intranet access control lists
-
-acl loopback {
-    127.0.0.1;
-};
-
-acl lan {
-    // the cache host IP should not be part of regular lan ACL
-    !10.0.0.88;
-    // private IPv4 address spaces
-    10.0.0.0/8;
-    172.16.0.0/12;
-    192.168.0.0/16;
-};
-
-acl lannocache {
-   // counterpart of earlier statement: cache host needs proper unspoofed name resolution
-   10.0.0.88;
-};
-
-
-// EOF

etc/bind/named.conf.ads

-// this can be used to ban ads servers content from generating trafic on you network
-// add a file /etc/cron.weekly/update-bind-ads-blocks if you want to use it with content like
-//
-//  #!/bin/sh
-//  /usr/local/bin/update-bind-ads-blocks.pl > /etc/bind/named.conf.ads
-//  /etc/init.d/bind9 reload 2>/dev/null 1>/dev/null
-//
-// It will overwrite this file.

etc/bind/named.conf.cache

-// build by ./named.conf.cache-rebuild.sh
-// re-run it commenting relevant domains if you dont cache them all
-zone "cs.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content1.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content2.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content3.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content4.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content5.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content6.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content7.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content8.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content9.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "hsar.steampowered.com.edgesuite.net" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "akamai.steamstatic.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "content-origin.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "client-download.steampowered.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "steampipe.steamcontent.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "steamcontent.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "http.debian.net" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "ftp.fr.debian.org" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "ftp.debian.org" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "security.debian.org" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "packages.devuan.org" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "fr.archive.ubuntu.com" { type master; notify no; file "/etc/bind/db.cache"; };
-zone "security.ubuntu.com" { type master; notify no; file "/etc/bind/db.cache"; };
-// EOF

etc/bind/named.conf.local_ref

-// Required since https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=820056
-// bind people decided you cannot have a single file along with allow-update in several
-// zones, you need now to use in-view to refer to the first time this file with allow-update
-// was defined in a zone
-
-zone "YOURZONE.LAN" {
-      in-view "loopback";
-};
-
-zone "1.168.192.in-addr.arpa" {
-  in-view "loopback";
-  
-};
-
-zone "0.0.10.in-addr.arpa" {
-  in-view "loopback";
- 
-};
-
-// EOF
\ No newline at end of file

etc/bind/named.conf.views

-// extra layer handling views
-// every zones must be included here, none directly within named.conf
-// only this file must be included in named.conf with
-//     include "/etc/bind/named.conf.views";
-
-// clients are set in named.conf.acl
-include "/etc/bind/named.conf.acl";
-
-// loopback view, for the server itself
-view "loopback" {
-     match-clients { loopback; };
-     allow-recursion { any; };
-     allow-query { any; };
-     allow-query-cache { any; };
-
-     // web queries
-     include "/etc/bind/named.conf.default-zones";
-     // lan queries
-     include "/etc/bind/named.conf.local";
-     // ads to devnull queries
-     include "/etc/bind/named.conf.ads";
-};
-
-// otherwise local network area
-view "lan" {
-     match-clients { lan; };
-     allow-recursion { any; };
-     allow-query { any; };
-     allow-query-cache { any; };
-
-     // web queries
-     include "/etc/bind/named.conf.default-zones";
-     // lan queries (odd in-view file since bind9 no longer allows a file to be updated in two places)     
-     include "/etc/bind/named.conf.local_ref";
-     // spoofed to local queries
-     include "/etc/bind/named.conf.cache";
-     // ads to devnull queries
-     include "/etc/bind/named.conf.ads";
-};
-
-// local network area without cache, for host that will get unspoofed name resolution
-// (needs to be set up one by one in named.conf.acl)
-view "lannocache" {
-     match-clients { lannocache; };
-     allow-recursion { any; };
-     allow-query { any; };
-     allow-query-cache { any; };
-     
-     // web queries
-     include "/etc/bind/named.conf.default-zones";
-     // lan queries (odd in-view file since bind9 no longer allows a file to be updated in two places)   
-     include "/etc/bind/named.conf.local_ref";
-     // ads to devnull queries
-     include "/etc/bind/named.conf.ads";
-};
-
-
-// if there was no match earlier, it means that we are coming from unknown network
-// that's odd and would need to be investigated
-// gives very limited access
-view "unknown" {
-     // retains default allow- settings
-
-     // web queries
-     include "/etc/bind/named.conf.default-zones";
-     // ads to devnull queries
-     include "/etc/bind/named.conf.ads";
-};
-
-// EOF

etc/dhcp/dhclient-exit-hooks.d/bind

-#!/bin/bash
-# Primitive script to generrate named.conf.options on the fly
-# Bind DNS cache need forwarders similar to the content of resolv.conf
-CONFFILE=/etc/bind/named.conf.options
-
-if [ -n "$new_domain_name_servers" ] || [ -n "$new_ip_address" ] ; then
-    
-    # change only if we new have DNS, new IP address or no conffile
-    if [ "$new_domain_name_servers" != "$old_domain_name_servers" ] ||
-	   [ "$new_ip_address" != "$old_ip_address" ] ||
-	   [ ! -e $CONFFILE ]; then
-	
-        echo "// DO NOT EDIT, automatically generated by $0
-// (IP changed from $old_ip_address to $new_ip_address) 
-// (DNS changed from $old_domain_name_servers to $new_domain_name_servers) 
-// `date`
-options {
-        directory \"/var/cache/bind\";
-
-        // If there is a firewall between you and nameservers you want
-        // to talk to, you may need to fix the firewall to allow multiple
-        // ports to talk.  See http://www.kb.cert.org/vuls/id/800113
-
-        // If your ISP provided one or more IP addresses for stable 
-        // nameservers, you probably want to use them as forwarders.  
-        forward first;
-        forwarders {" > $CONFFILE
-         
-
-        # add valid forwarders
-        for server in $new_domain_name_servers; do
-            # (verbose) skip local ips
-            if [ ! -n "`ifconfig | grep ":$server "`" ]; then 
-                echo "                $server;" >> $CONFFILE
-            else 
-                echo "                //LOCAL IP, skip it: $server;" >> $CONFFILE
-
-            fi
-        done
-
-        echo "        };
-
-        auth-nxdomain no;    # conform to RFC1035
-
-        // stick to ipv4 only
-        listen-on-v6 { none; };
-        // listen to any interface but this public one
-        listen-on { !$new_ip_address; any; };
-
-        // optional: select ACL to accept requests
-        //allow-query { loopback; lan; lannocache; };
-        // optional: select ACL to allow recursive queries
-        //allow-recursion { loopback; lan; lannocache; };
-
-        // do not make public version of BIND
-        version none;
-};
-
-// EOF" >> $CONFFILE
-        
-	# now reload bind
-	# (this may be useles because another script may do that already)
-	invoke-rc.d bind9 restart >/dev/null 2>&1
-	echo "$CONFFILE updated"
-    fi
-fi

usr/local/bin/update-bind-ads-block.pl → etc/powerdns/redirect-ads-rebuild.pl

 #!/usr/bin/perl -w
+#
+# Copyright (c) 2017 Mathieu Roy <yeupou--gnu.org>
+#      http://yeupou.wordpress.com
+#
+# modified version  of:
+# 
 # http://prefetch.net/blog/index.php/2006/05/27/using-bind-to-reduce-ad-server-content/
 # run by /etc/cron.weekly/update-bind-ads-block > /etc/bind/named.conf.ads
 #
@@ -50,49 +56,40 @@
 #   along with this program; if not, write to the Free Software
 #   Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
+use strict;
+use Fcntl ':flock';
+
+# disallow concurrent run
+open(LOCK, "< $0") or die "Failed to ask lock. Exiting";
+flock(LOCK, LOCK_EX | LOCK_NB) or die "Unable to lock. This daemon is already alive. Exiting";
+
+open(OUT, "> redirect-ads.lua");
 
 # You can choose between wget or curl. Both rock!
 # my $snagger = "curl -q";
 my $snagger = "wget -q -O - ";
 
-# If you want the "zone" entries appended to an existing named.conf,
-# specify it here. The entries will be appended with a timestamp.
-# my $named = "named.conf";
-
 # List of URLs to find ad servers.
-my @urls = ( "http://pgl.yoyo.org/adservers/serverlist.php?showintro=0;hostformat=hosts" );
-
-# If $named isn't NULL, dump it before the domains
-if ( defined $named ) {
-    open(NAMED, $named);
-
-    while ( <NAMED> ) {
-
-        if ( $_ !~ /Added domains on/ ) {
-             print $_;
-         } else {
-             last;
-         }
-    }
-}
+my @urls = ("http://pgl.yoyo.org/adservers/serverlist.php?showintro=0;hostformat=one-line;mimetype=plaintext");
 
+print OUT "return{\n";
 # Grab the list of domains and add them to the realm file
-foreach $url (@urls) {
-
+foreach my $url (@urls) {
     # Open the curl command
-    open(CURL, "$snagger $url |") || die "Cannot execute $snagger: $@\n";
+    open(CURL, "$snagger \"$url\" |") || die "Cannot execute $snagger: $@\n";
 
-    printf( "// *** Added domains on %s *** //\n\n", scalar localtime);
+    printf OUT ("--- Added domains on %s --\n", scalar localtime);
 
     while (<CURL>) {
-         chomp;
+	next if /^#/;
+	next if /^$/;
+	chomp();
+	foreach my $domain (split(",")) {
+	    print OUT "\"$domain\",\n";
 
-         # Should take care of host files in the following format: name
-         if ( $_ =~ /^[0-9a-zA-Z\-]+\.[0-9a-zA-Z\-]+$/) {
-              printf("zone \"%s\" { type master\; notify no\;", $_);
-              print  " file \"/etc/bind/db.ads\"\; }\;\n";
-         }
+	}
     }
 }
+print OUT "}\n";
 
 # EOF

etc/powerdns/redirect-blacklisted.lua

+return{
+"gfe.nvidia.com",
+}

etc/bind/named.conf.cache-rebuild.sh → etc/powerdns/redirect-cached-rebuild.sh

 #!/bin/sh
 
-#FOR DIFFERENT CACHE FILES PER NETWORK, uncomment#DNSS="192.168.1 10.0.0 10.0.1"
 DOMAINS=""
 
 # comment this if you dont cache steam
@@ -14,15 +13,13 @@ DOMAINS="$DOMAINS packages.devuan.org amprolla.devuan.org"
 # comment this if you dont cache ubuntu
 DOMAINS="$DOMAINS fr.archive.ubuntu.com security.ubuntu.com"
 
-dns=""
-#FOR DIFFERENT CACHE FILES PER NETWORK, uncomment#for dns in $DNSS; do
-    out=named.conf.cache$dns
-    echo "// build by ${0}" > $out
-    echo "// re-run it commenting relevant domains if you dont cache them all" >> $out
-    for domain in $DOMAINS; do
-	echo zone \"$domain\"  \{ type master\; notify no\; file \"/etc/bind/db.cache$dns\"\; \}\; >> $out
-    done
-    echo "// EOF" >> $out
-#FOR DIFFERENT CACHE FILES PER NETWORK, uncomment#done
+out=redirect-cached.lua
+echo "-- build by ${0}" > $out
+echo "-- re-run it commenting relevant domains if you dont cache them all" >> $out
+echo "return{" >> $out
+for domain in $DOMAINS; do
+    echo \"$domain\", >> $out
+done
+echo "}" >> $out
     
 # EOF

etc/powerdns/redirect.lua

+-- IPv4 only script
+-- Copyright (c) 2017 Mathieu Roy <yeupou--gnu.org>
+--                 http://yeupou.wordpress.com
+--
+-- This program is free software; you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License as published by
+-- the Free Software Foundation; either version 2 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License
+-- along with this program; if not, write to the Free Software
+-- Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+-- USA
+
+-- cached servers
+cached = newDS()
+cachedest = "10.0.0.88"
+
+-- ads kill list
+ads = newDS()
+adsdest = "127.0.0.1"
+
+-- hand maintained black list
+blacklisted = newDS()
+blacklistdest = "127.0.0.1"
+
+function preresolve(dq)
+   -- DEBUG
+   --pdnslog("Got question for "..dq.qname:toString().." from "..dq.remoteaddr:toString().." to "..dq.localaddr:toString(), pdns.loglevels.Error)
+   
+   -- handmade domains blacklist
+   if(blacklisted:check(dq.qname)) then
+      if(dq.qtype == pdns.A) then
+	 dq:addAnswer(dq.qtype, blacklistdest)
+	 return true
+      end
+   end
+   
+   -- spam/ads domains
+   if(ads:check(dq.qname)) then
+      if(dq.qtype == pdns.A) then
+	 dq:addAnswer(dq.qtype, adsdest)
+	 return true
+      end
+   end						
+   
+   -- cached domains
+   if(not cached:check(dq.qname)) then
+      -- not cached
+      return false
+   else
+      -- cached: variable answer
+      dq.variable = true
+      -- request coming from the cache itself
+      if(dq.remoteaddr:equal(newCA(cachedest))) then
+	 return false
+      end
+      --  redirect to the cache
+      if(dq.qtype == pdns.A) then
+	 dq:addAnswer(dq.qtype, cachedest)
+      end
+   end
+   
+   return true
+end
+
+cached:add(dofile("/etc/powerdns/redirect-cached.lua"))
+ads:add(dofile("/etc/powerdns/redirect-ads.lua"))
+blacklisted:add(dofile("/etc/powerdns/redirect-blacklisted.lua"))
+
+-- EOF