From 87ec92d1ed7cf94b1b17248728e59770c242570c Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Wed, 19 Dec 2018 17:33:29 +0100 Subject: [PATCH] luci-app-adblock: sync with release 3.6.0 * add adblock dns query reporting via tcpdump (see readme for details) * fix tld compression on low memory systems (< 64 MB) * fix various small issues Signed-off-by: Dirk Brenken --- .../luasrc/controller/adblock.lua | 76 +++++- .../luasrc/model/cbi/adblock/overview_tab.lua | 159 +++++++----- .../luasrc/view/adblock/adblock_css.htm | 10 +- .../luasrc/view/adblock/report.htm | 244 ++++++++++++++++++ .../luasrc/view/adblock/runtime.htm | 4 +- 5 files changed, 420 insertions(+), 73 deletions(-) create mode 100644 applications/luci-app-adblock/luasrc/view/adblock/report.htm diff --git a/applications/luci-app-adblock/luasrc/controller/adblock.lua b/applications/luci-app-adblock/luasrc/controller/adblock.lua index 4c7724471..0cfb8fc80 100644 --- a/applications/luci-app-adblock/luasrc/controller/adblock.lua +++ b/applications/luci-app-adblock/luasrc/controller/adblock.lua @@ -16,7 +16,8 @@ function index() end entry({"admin", "services", "adblock"}, firstchild(), _("Adblock"), 30).dependent = false entry({"admin", "services", "adblock", "tab_from_cbi"}, cbi("adblock/overview_tab", {hideresetbtn=true, hidesavebtn=true}), _("Overview"), 10).leaf = true - entry({"admin", "services", "adblock", "log"}, template("adblock/logread"), _("View Logfile"), 20).leaf = true + entry({"admin", "services", "adblock", "report"}, template("adblock/report"), _("DNS Query Report"), 20).leaf = true + entry({"admin", "services", "adblock", "log"}, template("adblock/logread"), _("Logfile"), 30).leaf = true entry({"admin", "services", "adblock", "advanced"}, firstchild(), _("Advanced"), 100) entry({"admin", "services", "adblock", "advanced", "blacklist"}, form("adblock/blacklist_tab"), _("Edit Blacklist"), 110).leaf = true entry({"admin", "services", "adblock", "advanced", "whitelist"}, form("adblock/whitelist_tab"), _("Edit Whitelist"), 120).leaf = true @@ -24,17 +25,49 @@ function index() entry({"admin", "services", "adblock", "advanced", "query"}, template("adblock/query"), _("Query domains"), 140).leaf = true entry({"admin", "services", "adblock", "advanced", "result"}, call("queryData"), nil, 150).leaf = true entry({"admin", "services", "adblock", "logread"}, call("logread"), nil).leaf = true - entry({"admin", "services", "adblock", "status"}, call("status_update"), nil).leaf = true + entry({"admin", "services", "adblock", "status_update"}, call("status_update"), nil).leaf = true + entry({"admin", "services", "adblock", "report_json"}, call("report_json"), nil).leaf = true + entry({"admin", "services", "adblock", "report_text"}, call("report_text"), nil).leaf = true entry({"admin", "services", "adblock", "action"}, call("adb_action"), nil).leaf = true end -function adb_action(name) +function adb_action(name, domain) if name == "do_suspend" then luci.sys.call("/etc/init.d/adblock suspend >/dev/null 2>&1") elseif name == "do_resume" then luci.sys.call("/etc/init.d/adblock resume >/dev/null 2>&1") elseif name == "do_refresh" then luci.sys.call("/etc/init.d/adblock reload >/dev/null 2>&1") + local pid_file = "/var/run/adblock.pid" + if nixio.fs.access(pid_file) then + repeat + nixio.nanosleep(1) + until nixio.fs.readfile(pid_file) == "" + end + elseif name == "do_report" then + luci.sys.call("/etc/init.d/adblock report false >/dev/null 2>&1") + local rep_dir = uci:get("adblock", "extra", "adb_repdir") or "/tmp" + repeat + nixio.nanosleep(1) + until not nixio.fs.access(rep_dir.. "/adb_report.raw") + elseif name == "add_blacklist" then + local file = uci:get("adblock", "blacklist", "adb_src") or "/etc/adblock/adblock.blacklist" + if nixio.fs.access(file) then + local blacklist = nixio.fs.readfile(file) + if not string.find(blacklist, domain, 1, plain) + then + nixio.fs.writefile(file, blacklist.. domain.. "\n") + end + end + elseif name == "add_whitelist" then + local file = uci:get("adblock", "global", "adb_whitelist") or "/etc/adblock/adblock.whitelist" + if nixio.fs.access(file) then + local whitelist = nixio.fs.readfile(file) + if not string.find(whitelist, domain, 1, plain) + then + nixio.fs.writefile(file, whitelist.. domain.. "\n") + end + end end luci.http.prepare_content("text/plain") luci.http.write("0") @@ -53,6 +86,43 @@ function status_update() end end +function report_json() + local rep_dir + local rep_file + local content + + rep_dir = uci:get("adblock", "extra", "adb_repdir") or "/tmp" + rep_file = rep_dir.. "/adb_report.json" + http.prepare_content("application/json") + + if nixio.fs.access(rep_file) then + content = json.parse(nixio.fs.readfile(rep_file) or "") + http.write_json(content) + else + http.write_json("{}") + end +end + +function report_text() + local file + local rep_dir + local rep_file + local content + + rep_dir = uci:get("adblock", "extra", "adb_repdir") or "/tmp" + rep_file = rep_dir.. "/adb_report" + http.prepare_content("text/plain") + + if nixio.fs.access(rep_file) then + file = io.open(rep_file, "r") + content = file:read("*all") + file:close() + http.write(content) + else + http.write("") + end +end + function logread() local content diff --git a/applications/luci-app-adblock/luasrc/model/cbi/adblock/overview_tab.lua b/applications/luci-app-adblock/luasrc/model/cbi/adblock/overview_tab.lua index 2ecaaab72..819a77216 100644 --- a/applications/luci-app-adblock/luasrc/model/cbi/adblock/overview_tab.lua +++ b/applications/luci-app-adblock/luasrc/model/cbi/adblock/overview_tab.lua @@ -1,14 +1,16 @@ -- Copyright 2017-2018 Dirk Brenken (dev@brenken.org) -- This is free software, licensed under the Apache License, Version 2.0 -local fs = require("nixio.fs") -local uci = require("luci.model.uci").cursor() -local util = require("luci.util") -local dump = util.ubus("network.interface", "dump", {}) +local fs = require("nixio.fs") +local uci = require("luci.model.uci").cursor() +local util = require("luci.util") +local net = require "luci.model.network".init() +local sys = require("luci.sys") +local devices = sys.net:devices() m = Map("adblock", translate("Adblock"), translate("Configuration of the adblock package to block ad/abuse domains by using DNS. ") - .. translatef("For further information " + ..translatef("For further information " .. "" .. "check the online documentation", "https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md")) @@ -22,7 +24,7 @@ o1.rmempty = false o2 = s:option(ListValue, "adb_dns", translate("DNS Backend (DNS Directory)"), translate("List of supported DNS backends with their default list export directory. ") - .. translate("To overwrite the default path use the 'DNS Directory' option in the extra section below.")) + ..translate("To overwrite the default path use the 'DNS Directory' option in the extra section below.")) o2:value("dnsmasq", "dnsmasq (/tmp)") o2:value("unbound", "unbound (/var/lib/unbound)") o2:value("named", "named (/var/lib/bind)") @@ -44,14 +46,18 @@ o3.rmempty = false o4 = s:option(ListValue, "adb_trigger", translate("Startup Trigger"), translate("List of available network interfaces. Usually the startup will be triggered by the 'wan' interface. ") - .. translate("Choose 'none' to disable automatic startups, 'timed' to use a classic timeout (default 30 sec.) or select another trigger interface.")) + ..translate("Choose 'none' to disable automatic startups, 'timed' to use a classic timeout (default 30 sec.) or select another trigger interface.")) o4:value("none") o4:value("timed") -if dump then - local i, v - for i, v in ipairs(dump.interface) do - if v.interface ~= "loopback" then - o4:value(v.interface) +for _, dev in ipairs(devices) do + if dev ~= "lo" then + local iface = net:get_interface(dev) + if iface then + iface = iface:get_networks() or {} + for k, v in pairs(iface) do + iface[k] = iface[k].sid + o4:value(iface[k], iface[k].. " (" ..dev.. ")") + end end end end @@ -94,90 +100,117 @@ e = m:section(NamedSection, "extra", "adblock", translate("Extra Options"), e1 = e:option(Flag, "adb_debug", translate("Verbose Debug Logging"), translate("Enable verbose debug logging in case of any processing error.")) -e1.default = e1.disabled e1.rmempty = false e2 = e:option(Flag, "adb_nice", translate("Low Priority Service"), translate("Set the nice level to 'low priority' and the adblock background processing will take less resources from the system. ") ..translate("This change requires a manual service stop/re-start to take effect.")) -e2.default = e2.disabled e2.disabled = "0" e2.enabled = "10" e2.rmempty = false e3 = e:option(Flag, "adb_forcedns", translate("Force Local DNS"), translate("Redirect all DNS queries from 'lan' zone to the local resolver, apply to udp and tcp protocol on ports 53, 853 and 5353.")) -e3.default = e3.disabled e3.rmempty = false -e4 = e:option(Flag, "adb_forcesrt", translate("Force Overall Sort"), - translate("Enable memory intense overall sort / duplicate removal on low memory devices (< 64 MB free RAM)")) -e4.default = e4.disabled -e4.rmempty = false - -e5 = e:option(Flag, "adb_backup", translate("Enable Blocklist Backup"), +e4 = e:option(Flag, "adb_backup", translate("Enable Blocklist Backup"), translate("Create compressed blocklist backups, they will be used in case of download errors or during startup in backup mode.")) -e5.default = e5.disabled -e5.rmempty = false +e4.rmempty = false -e6 = e:option(Value, "adb_backupdir", translate("Backup Directory"), - translate("Target directory for adblock backups. Please use only non-volatile disks, e.g. an external usb stick.")) -e6:depends("adb_backup", 1) -e6.datatype = "directory" -e6.default = "/mnt" +e5 = e:option(Value, "adb_backupdir", translate("Backup Directory"), + translate("Target directory for adblock backups. Please use only a non-volatile disk, e.g. an external usb stick.")) +e5:depends("adb_backup", 1) +e5.datatype = "directory" +e5.default = "/mnt" e5.rmempty = true -e7 = e:option(Flag, "adb_backup_mode", translate("Backup Mode"), +e6 = e:option(Flag, "adb_backup_mode", translate("Backup Mode"), translate("Do not automatically update blocklists during startup, use blocklist backups instead.")) -e7:depends("adb_backup", 1) -e7.default = e7.disabled -e7.rmempty = true - -e8 = e:option(Value, "adb_maxqueue", translate("Max. Download Queue"), - translate("Size of the download queue to handle downloads & list processing in parallel (default '4'). ") - .. translate("For further performance improvements you can raise this value, e.g. '8' or '16' should be safe.")) -e8.default = 4 -e8.datatype = "range(1,32)" +e6:depends("adb_backup", 1) +e6.rmempty = true + +e7 = e:option(Value, "adb_maxqueue", translate("Max. Download Queue"), + translate("Size of the download queue to handle downloads & list processing in parallel (default '8'). ") + ..translate("For further performance improvements you can raise this value, e.g. '8' or '16' should be safe.")) +e7.default = 8 +e7.datatype = "range(1,32)" +e7.rmempty = false + +e8 = e:option(Flag, "adb_report", translate("Enable DNS Query Report"), + translate("Gather dns related network traffic via tcpdump to provide a DNS Query Report on demand. ") + ..translate("Please note: this needs manual 'tcpdump-mini' package installation.")) e8.rmempty = false -e9 = e:option(Flag, "adb_jail", translate("'Jail' Blocklist Creation"), - translate("Builds an additional 'Jail' list (/tmp/adb_list.jail) to block access to all domains except those listed in the whitelist file. ") - .. translate("You can use this restrictive blocklist manually e.g. for guest wifi or kidsafe configurations.")) -e9.default = e9.disabled -e9.rmempty = true - -e9 = e:option(Flag, "adb_dnsflush", translate("Flush DNS Cache"), - translate("Flush DNS Cache after adblock processing.")) -e9.default = e9.disabled +e9 = e:option(Value, "adb_repdir", translate("Report Directory"), + translate("Target directory for dns related report files. Please use preferably a non-volatile disk, e.g. an external usb stick.")) +e9:depends("adb_report", 1) +e9.datatype = "directory" +e9.default = "/tmp" e9.rmempty = true e10 = e:option(Flag, "adb_notify", translate("Email Notification"), translate("Send notification emails in case of a processing error or if domain count is ≤ 0. ") - .. translate("Please note: this needs additional 'msmtp' package installation and setup.")) -e10.default = e10.disabled + .. translate("Please note: this needs manual 'msmtp' package installation and setup.")) e10.rmempty = true -e11 = e:option(Value, "adb_notifycnt", translate("Email Notification Count"), +-- Optional Extra Options + +e20 = e:option(Flag, "adb_jail", translate("'Jail' Blocklist Creation"), + translate("Builds an additional 'Jail' list (/tmp/adb_list.jail) to block access to all domains except those listed in the whitelist file. ") + .. translate("You can use this restrictive blocklist e.g. for guest wifi or kidsafe configurations.")) +e20.optional = true +e20.default = nil + +e21 = e:option(Value, "adb_notifycnt", translate("Email Notification Count"), translate("Raise the minimum email notification count, to get emails if the overall count is less or equal to the given limit (default 0), ") .. translate("e.g. to receive an email notification with every adblock update set this value to 150000.")) -e11.default = 0 -e11.datatype = "min(0)" -e11.optional = true +e21.default = 0 +e21.datatype = "min(0)" +e21.optional = true -e12 = e:option(Value, "adb_dnsdir", translate("DNS Directory"), +e22 = e:option(Value, "adb_dnsdir", translate("DNS Directory"), translate("Target directory for the generated blocklist 'adb_list.overall'.")) -e12.datatype = "directory" -e12.optional = true +e22.datatype = "directory" +e22.optional = true -e13 = e:option(Value, "adb_whitelist", translate("Whitelist File"), +e23 = e:option(Value, "adb_whitelist", translate("Whitelist File"), translate("Full path to the whitelist file.")) -e13.datatype = "file" -e13.default = "/etc/adblock/adblock.whitelist" -e13.optional = true +e23.datatype = "file" +e23.default = "/etc/adblock/adblock.whitelist" +e23.optional = true -e14 = e:option(Value, "adb_triggerdelay", translate("Trigger Delay"), +e24 = e:option(Value, "adb_triggerdelay", translate("Trigger Delay"), translate("Additional trigger delay in seconds before adblock processing begins.")) -e14.datatype = "range(1,60)" -e14.optional = true +e24.datatype = "range(1,60)" +e24.optional = true + +e25 = e:option(Flag, "adb_dnsflush", translate("Flush DNS Cache"), + translate("Flush DNS Cache after adblock processing.")) +e25.optional = true +e25.default = nil + +e26 = e:option(ListValue, "adb_repiface", translate("Report Interface"), + translate("Reporting interface used by tcpdump (default 'br-lan').")) +for _, dev in ipairs(devices) do + if dev ~= "lo" then + e26:value(dev) + end +end +e26.optional = true + +e28 = e:option(Value, "adb_repchunkcnt", translate("Report Chunk Count"), + translate("Report chunk count used by tcpdump (default '5').")) +e28.datatype = "range(1,10)" +e28.optional = true + +e29 = e:option(Value, "adb_repchunksize", translate("Report Chunk Size"), + translate("Report chunk size used by tcpdump in MB (default '1').")) +e29.datatype = "range(1,10)" +e29.optional = true + +e30 = e:option(Flag, "adb_forcesrt", translate("Force Overall Sort"), + translate("Enable memory intense overall sort / duplicate removal on low memory devices (< 64 MB free RAM)")) +e30.optional = true +e30.default = nil return m diff --git a/applications/luci-app-adblock/luasrc/view/adblock/adblock_css.htm b/applications/luci-app-adblock/luasrc/view/adblock/adblock_css.htm index e6460122e..5467c8763 100644 --- a/applications/luci-app-adblock/luasrc/view/adblock/adblock_css.htm +++ b/applications/luci-app-adblock/luasrc/view/adblock/adblock_css.htm @@ -1,8 +1,8 @@