#!/bin/sh
#
# license: Standard BSD2CLAUSE (BSD 2-clause Simplified License),
# Please read from the web.
#
            
# This script process the host file maintained at hphosts.gt500.org 
# which provides a compressed hosts file that contains 766,963 fqdn.
# hp.hosts = multiple categories

# This script process the host files maintained at the french site
# http://rlwpx.free.fr/WPFF/hosts.htm
# hmis.7z = anti-miscellaneous stuff that don't fall in the other categories.
# hrsk.7z = anti-malware
# hsex.7z = anti-sex sites
# htrc.7z = anti-stalking
# hpub.7z = anti-advertising
            
            
# Note to user:
# All these downloaded files are compressed files done with the 7zip utility.
# The tar command gets an error un-compressing the hpub.7z file
# which hangs this script.
# The meaning of the hpub switch has two setting. You choose setting.
#           
# hpub_switch="y" means process the hpub file and use the 7zip utility
#                 for un-compressing all the files and install the 7zip
#                 utility on the fly if not installed all ready.
#                 Resulting size of the formated output file
#                 is around 727,661 unique blocked domains.
#           
# hpub_switch="n" the default means DON'T process the hpub file and
#                 use tar command to un-compress all the files.
#                 Resulting size of the formated output file
#                 is around 694,721 unique blocked domains.
#           
            
  hpub_switch="n"
            
  if [ ${hpub_switch} = "y" ]; then
    if [ ! -e "/usr/local/bin/7za" ]; then
      pkg install p7zip
    fi      
  fi        
            
            
# For whatever DNS server port/package your using do this
# Un-comment for which ever DNS server you are using.
# Do same at end of this script to select correct output format.
  DNS="unbound"
#  DNS="named"
#  DNS="dnsmasq"
  path="/usr/local/etc/$DNS"
#  path="/tmp"
            
# If using the local_unbound version un-comment next 2 lines.
#  DNS="local_unbound"
#  path="/var/unbound"
            
            
  op="$path/fqdn-list"
  if [ -f "${op}" ]; then
    mv $op "$op-prev2"
    truncate -s 0 $op
  else
    truncate -s 0 $op
  fi

  op="$path/fqdn-list"
  truncate -s 0 $op
            
              
  DL_DIR="$path/fqdn2_downloaded"
  if [ ! -d "$DL_DIR" ]; then
     mkdir -p "$DL_DIR"
  else      
    rm -rf   "$DL_DIR"
    mkdir -p "$DL_DIR"
  fi            
            
            
  UZ_DIR="$path/fqdn2_unziped"
  if [ ! -d "$UZ_DIR" ]; then
    mkdir -p "$UZ_DIR"
  else      
    rm -rf   "$UZ_DIR"
    mkdir -p "$UZ_DIR"
  fi        
            
            
  CLEAN_DIR="$path/fqdn2_clean"
  if [ ! -d "$CLEAN_DIR" ]; then
    mkdir -p "$CLEAN_DIR"
  else      
    rm -rf   "$CLEAN_DIR"
    mkdir -p "$CLEAN_DIR"
  fi        
            
            
  ND2_DIR="$path/fqdn2_nodups"
  if [ ! -d "$ND2_DIR" ]; then
    mkdir -p "$ND2_DIR"
  else      
    rm -rf   "$ND2_DIR"
    mkdir -p "$ND2_DIR"
  fi        
            
            
  ND1_DIR="$path/fqdn1_nodups"
            
#  FETCH="/usr/bin/fetch -q"
  FETCH="/usr/bin/fetch "
            
  cd "$DL_DIR"
            
  echo " "  
  echo "Downloading host file provider FQDN files."
  echo " "  
            
  $FETCH "http://hphosts.gt500.org/hosts.zip"
  $FETCH "http://rlwpx.free.fr/WPFF/hmis.7z"
  $FETCH "http://rlwpx.free.fr/WPFF/hrsk.7z"
  $FETCH "http://rlwpx.free.fr/WPFF/hsex.7z"
  $FETCH "http://rlwpx.free.fr/WPFF/htrc.7z"
            
  if [ ${hpub_switch} = "y" ]; then
    $FETCH "http://rlwpx.free.fr/WPFF/hpub.7z"
  fi        
            
  #         
  # Fetched files are compressed using 7z. Use tar to uncompress them.
  #         
  echo " "  
  echo "Uncompressing the downloaded files now."
  echo " "  
            
  list=`ls`
  cd "$UZ_DIR"
  if [ "${list}" ]; then
    # There are fetched files.
    for filename in ${list}; do
      if [ ${hpub_switch} = "y" ]; then
        /usr/local/bin/7za x "${DL_DIR}/${filename}"
      else  
        /usr/bin/tar xvf "${DL_DIR}/${filename}"
      fi    
    done    
  else      
    echo "No fetched fles."
    exit 2  
  fi        
            
  # The hosts.zip contained two extra files that we have to remove now
  # 
  /bin/rm "${UZ_DIR}/readme.txt"
  /bin/rm "${UZ_DIR}/hosts.txt.asc"
            
            
  # pgm1 reads the downloaded file creates a output file containing 
  # host name only. pgm2 sorts the host name only file right to left 
  # then sorts again to drop dups.
  # 
  echo " "  
  echo "Sanitizing and dropping dups from the uncompressed files."        
  echo " "  
            
  pgm1="/usr/local/bin/dns2blackhole-sanitize"
  pgm2="/usr/local/bin/dns2blackhole-sort-fqdn"
  CAT="/bin/cat"
            
  cd "$UZ_DIR"
            
  list=`ls` 
  if [ "${list}" ]; then
    # There are downloaded files.
    for filename in ${list}; do
      ${pgm1} "${UZ_DIR}/${filename}" "${CLEAN_DIR}/${filename}" 
      ${CAT} "${CLEAN_DIR}/${filename}" | ${pgm2} > "${ND2_DIR}/${filename}"
    done    
  else      
    echo "No downloaded."
    exit 2  
  fi        
            
            
  wl="$path/whitelist.txt"
  pgm3="/usr/local/bin/dns2blackhole-whitelist"
            
  echo " "  
  echo "Applying whitelist, merging with update1 list" 
  echo " "  
            
# For unbound, Process whitelist file if it exists, and
# combine the two lists into single list.
  if [ -f "${wl}" ]; then
    ${CAT} ${ND1_DIR}/* ${ND2_DIR}/* | ${pgm2} | ${pgm3} "${wl}" | awk '
      { print "local-zone:", $1 " always_nxdomain" }' > $op
  else      
    ${CAT} ${ND1_DIR}/* ${ND2_DIR}/* | ${pgm2} | awk '
      { print "local-zone:", $1 " always_nxdomain" }' > $op
  fi        
            
            
# For dnsmasq, Process whitelist file if it exists.
#  if [ -f "${wl}" ]; then
#    ${CAT} ${ND1_DIR}/* ${ND2_DIR}/* | ${pgm2} | ${pgm3} "${wl}" | awk '
#      { print "address=/" $1 "/127.0.0.1" }' > $op
#  else     
#    ${CAT} ${ND1_DIR}/* ${ND2_DIR}/* | ${pgm2} | awk '
#      { print "address=/" $1 "/127.0.0.1" }' > $op
#  fi       
            
            
# For named, Process whitelist file if it exists.
#  if [ -f "${wl}" ]; then
#    ${CAT} ${ND1_DIR}/* ${ND2_DIR}/* | ${pgm2} | ${pgm3} "${wl}" | awk '
#      { print "zone \"" $1 "\" {type master; file \"/etc/bad.hosts\";};" }' > $op
#  else     
#    ${CAT} ${ND1_DIR}/* ${ND2_DIR}/* | ${pgm2} | awk '
#      { print "zone \"" $1 "\" {type master; file \"/etc/bad.hosts\";};" }' > $op
#  fi       

  service $DNS restart
            
