Merge pull request #1431 from pi-hole/development

v5.18

Author: PromoFaux

.github/workflows/build.yml

@@ -164,4 +164,4 @@ jobs:
         uses: softprops/action-gh-release@v1
         with:
           files: |
-            *
+            ${{ steps.download.outputs.download-path }}/*

CMakeLists.txt

@@ -11,6 +11,6 @@
 cmake_minimum_required(VERSION 2.8.12)
 project(PIHOLE_FTL C)
 
-set(DNSMASQ_VERSION pi-hole-2.87test8)
+set(DNSMASQ_VERSION pi-hole-v2.87rc1)
 
 add_subdirectory(src)

build.sh

@@ -49,7 +49,7 @@ cmake --build . -- -j $(nproc)
 # Otherwise, we simply copy the binary one level up
 if [[ -n "${install}" ]]; then
     echo "Installing pihole-FTL"
-    SUDO=$(which sudo)
+    SUDO=$(command -v sudo)
     ${SUDO} cmake --install .
 else
     echo "Copying compiled pihole-FTL binary to repository root"

src/api/api.c

@@ -1008,15 +1008,14 @@ void getAllQueries(const char *client_message, const int sock, const bool isteln
 			CNAME_domain = getCNAMEDomainString(query);
 		}
 
-		// Get ID of blocking regex, if applicable and permitted by privacy settings
-		int regex_idx = -1;
-		if ((query->status == QUERY_REGEX || query->status == QUERY_REGEX_CNAME) &&
-		    config.privacylevel < PRIVACY_HIDE_DOMAINS)
+		// Get domainlist table ID, if applicable and permitted by privacy settings
+		int domainlist_id = -1;
+		if (config.privacylevel < PRIVACY_HIDE_DOMAINS)
 		{
-			unsigned int cacheID = findCacheID(query->domainID, query->clientID, query->type);
+			unsigned int cacheID = findCacheID(query->domainID, query->clientID, query->type, false);
 			DNSCacheData *dns_cache = getDNSCache(cacheID, true);
 			if(dns_cache != NULL)
-				regex_idx = dns_cache->black_regex_idx;
+				domainlist_id = dns_cache->domainlist_id;
 		}
 
 		// Get IP of upstream destination, if applicable
@@ -1065,7 +1064,7 @@ void getAllQueries(const char *client_message, const int sock, const bool isteln
 				reply,
 				delay,
 				CNAME_domain,
-				regex_idx,
+				domainlist_id,
 				upstream_name,
 				upstream_port,
 				query->ede == -1 ? "" : get_edestr(query->ede));

src/database/gravity-db.c

@@ -15,13 +15,12 @@
 #include "../config.h"
 // logg()
 #include "../log.h"
-// match_regex()
-#include "../regex_r.h"
 // getstr()
 #include "../shmem.h"
 // SQLite3 prepared statement vectors
 #include "../vector.h"
 // log_subnet_warning()
+// logg_inaccessible_adlist
 #include "message-table.h"
 // getMACfromIP()
 #include "network-table.h"
@@ -210,11 +209,11 @@ bool gravityDB_reopen(void)
 	return gravityDB_open();
 }
 
-static char* get_client_querystr(const char* table, const char* groups)
+static char* get_client_querystr(const char *table, const char *column, const char *groups)
 {
 	// Build query string with group filtering
 	char *querystr = NULL;
-	if(asprintf(&querystr, "SELECT EXISTS(SELECT domain from %s WHERE domain = ? AND group_id IN (%s));", table, groups) < 1)
+	if(asprintf(&querystr, "SELECT %s from %s WHERE domain = ? AND group_id IN (%s);", column, table, groups) < 1)
 	{
 		logg("get_client_querystr(%s, %s) - asprintf() error", table, groups);
 		return NULL;
@@ -858,19 +857,14 @@ bool gravityDB_prepare_client_statements(clientsData *client)
 		return false;
 
 	// Prepare whitelist statement
-	// We use SELECT EXISTS() as this is known to efficiently use the index
-	// We are only interested in whether the domain exists or not in the
-	// list but don't case about duplicates or similar. SELECT EXISTS(...)
-	// returns true as soon as it sees the first row from the query inside
-	// of EXISTS().
 	if(config.debug & DEBUG_DATABASE)
 		logg("gravityDB_open(): Preparing vw_whitelist statement for client %s", clientip);
-	querystr = get_client_querystr("vw_whitelist", getstr(client->groupspos));
+	querystr = get_client_querystr("vw_whitelist", "id", getstr(client->groupspos));
 	sqlite3_stmt* stmt = NULL;
 	int rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
 	if( rc != SQLITE_OK )
 	{
-		logg("gravityDB_open(\"SELECT EXISTS(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
+		logg("gravityDB_open(\"SELECT(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
 		gravityDB_close();
 		return false;
 	}
@@ -880,11 +874,11 @@ bool gravityDB_prepare_client_statements(clientsData *client)
 	// Prepare gravity statement
 	if(config.debug & DEBUG_DATABASE)
 		logg("gravityDB_open(): Preparing vw_gravity statement for client %s", clientip);
-	querystr = get_client_querystr("vw_gravity", getstr(client->groupspos));
+	querystr = get_client_querystr("vw_gravity", "domain", getstr(client->groupspos));
 	rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
 	if( rc != SQLITE_OK )
 	{
-		logg("gravityDB_open(\"SELECT EXISTS(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
+		logg("gravityDB_open(\"SELECT(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
 		gravityDB_close();
 		return false;
 	}
@@ -894,11 +888,11 @@ bool gravityDB_prepare_client_statements(clientsData *client)
 	// Prepare blacklist statement
 	if(config.debug & DEBUG_DATABASE)
 		logg("gravityDB_open(): Preparing vw_blacklist statement for client %s", clientip);
-	querystr = get_client_querystr("vw_blacklist", getstr(client->groupspos));
+	querystr = get_client_querystr("vw_blacklist", "id", getstr(client->groupspos));
 	rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
 	if( rc != SQLITE_OK )
 	{
-		logg("gravityDB_open(\"SELECT EXISTS(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
+		logg("gravityDB_open(\"SELECT(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
 		gravityDB_close();
 		return false;
 	}
@@ -1146,7 +1140,7 @@ int gravityDB_count(const enum gravity_tables list)
 	return result;
 }
 
-static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, const char *listname)
+static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, const char *listname, int *domain_id)
 {
 	// Do not try to bind text to statement when database is not available
 	if(!gravityDB_opened && !gravityDB_open())
@@ -1180,19 +1174,21 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con
 		sqlite3_clear_bindings(stmt);
 		return LIST_NOT_AVAILABLE;
 	}
-	else if(rc != SQLITE_ROW)
+	else if(rc != SQLITE_ROW && rc != SQLITE_DONE)
 	{
-		// Any return code that is neither SQLITE_BUSY not SQLITE_ROW
-		// is a real error we should log
+		// Any return code that is neither SQLITE_BUSY nor SQLITE_ROW or
+		// SQLITE_DONE is an error we should log
 		logg("domain_in_list(\"%s\", %p, %s): Failed to perform step: %s",
 		     domain, stmt, listname, sqlite3_errstr(rc));
 		sqlite3_reset(stmt);
 		sqlite3_clear_bindings(stmt);
 		return LIST_NOT_AVAILABLE;
 	}
 
-	// Get result of query "SELECT EXISTS(...)"
-	const int result = sqlite3_column_int(stmt, 0);
+	// Get result of query (if available)
+	const int result = (rc == SQLITE_ROW) ? sqlite3_column_int(stmt, 0) : -1;
+	if(domain_id != NULL)
+		*domain_id = result;
 
 	if(config.debug & DEBUG_DATABASE)
 		logg("domain_in_list(\"%s\", %p, %s): %d", domain, stmt, listname, result);
@@ -1209,8 +1205,7 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con
 	sqlite3_clear_bindings(stmt);
 
 	// Return if domain was found in current table
-	// SELECT EXISTS(...) either returns 0 (false) or 1 (true).
-	return (result == 1) ? FOUND : NOT_FOUND;
+	return (rc == SQLITE_ROW) ? FOUND : NOT_FOUND;
 }
 
 void gravityDB_reload_groups(clientsData* client)
@@ -1268,17 +1263,7 @@ enum db_result in_whitelist(const char *domain, DNSCacheData *dns_cache, clients
 	// We have to check both the exact whitelist (using a prepared database statement)
 	// as well the compiled regex whitelist filters to check if the current domain is
 	// whitelisted.
-	enum db_result on_whitelist = domain_in_list(domain, stmt, "whitelist");
-
-	// For performance reasons, the regex evaluations is executed only if the
-	// exact whitelist lookup does not deliver a positive match. This is an
-	// optimization as the database lookup will most likely hit (a) more domains
-	// and (b) will be faster (given a sufficiently large number of regex
-	// whitelisting filters).
-	if(on_whitelist == NOT_FOUND)
-		on_whitelist = match_regex(domain, dns_cache, client->id, REGEX_WHITELIST, false) != -1;
-
-	return on_whitelist;
+	return domain_in_list(domain, stmt, "whitelist", &dns_cache->domainlist_id);
 }
 
 enum db_result in_gravity(const char *domain, clientsData *client)
@@ -1306,10 +1291,10 @@ enum db_result in_gravity(const char *domain, clientsData *client)
 	if(stmt == NULL)
 		stmt = gravity_stmt->get(gravity_stmt, client->id);
 
-	return domain_in_list(domain, stmt, "gravity");
+	return domain_in_list(domain, stmt, "gravity", NULL);
 }
 
-enum db_result in_blacklist(const char *domain, clientsData *client)
+enum db_result in_blacklist(const char *domain, DNSCacheData *dns_cache, clientsData *client)
 {
 	// If list statement is not ready and cannot be initialized (e.g. no
 	// access to the database), we return false to prevent an FTL crash
@@ -1334,7 +1319,7 @@ enum db_result in_blacklist(const char *domain, clientsData *client)
 	if(stmt == NULL)
 		stmt = blacklist_stmt->get(blacklist_stmt, client->id);
 
-	return domain_in_list(domain, stmt, "blacklist");
+	return domain_in_list(domain, stmt, "blacklist", &dns_cache->domainlist_id);
 }
 
 bool in_auditlist(const char *domain)
@@ -1345,7 +1330,7 @@ bool in_auditlist(const char *domain)
 		return false;
 
 	// We check the domain_audit table for the given domain
-	return domain_in_list(domain, auditlist_stmt, "auditlist") == FOUND;
+	return domain_in_list(domain, auditlist_stmt, "auditlist", NULL) == FOUND;
 }
 
 bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regexData *regex,
@@ -1407,3 +1392,43 @@ bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int n
 
 	return true;
 }
+
+void check_inaccessible_adlists(void)
+{
+
+	// check if any adlist was inaccessible in the last gravity run
+	// if so, gravity stored `status` in the adlist table with
+	// "3": List unavailable, Pi-hole used a local copy
+	// "4": List unavailable, there is no local copy available 
+
+	// Do not proceed when database is not available
+	if(!gravityDB_opened && !gravityDB_open())
+	{
+		logg("check_inaccessible_adlists(): Gravity database not available");
+		return;
+	}
+
+	const char *querystr = "SELECT id, address FROM adlist WHERE status IN (3,4) AND enabled=1";
+	
+	// Prepare query
+	sqlite3_stmt *query_stmt;
+	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &query_stmt, NULL);
+	if(rc != SQLITE_OK){
+		logg("check_inaccessible_adlists(): %s - SQL error prepare: %s", querystr, sqlite3_errstr(rc));
+		gravityDB_close();
+		return;
+	}
+
+	// Perform query
+	while((rc = sqlite3_step(query_stmt)) == SQLITE_ROW)
+	{
+		int id = sqlite3_column_int(query_stmt, 0);
+		const char *address = (const char*)sqlite3_column_text(query_stmt, 1);
+
+		// log to the message table
+		logg_inaccessible_adlist(id, address);
+	}
+
+	// Finalize statement
+	sqlite3_finalize(query_stmt);
+}

src/database/gravity-db.h

@@ -29,9 +29,10 @@ const char* gravityDB_getDomain(int *rowid);
 char* get_client_names_from_ids(const char *group_ids) __attribute__ ((malloc));
 void gravityDB_finalizeTable(void);
 int gravityDB_count(const enum gravity_tables list);
+void check_inaccessible_adlists(void);
 
 enum db_result in_gravity(const char *domain, clientsData *client);
-enum db_result in_blacklist(const char *domain, clientsData *client);
+enum db_result in_blacklist(const char *domain, DNSCacheData *dns_cache, clientsData *client);
 enum db_result in_whitelist(const char *domain, DNSCacheData *dns_cache, clientsData *client);
 bool in_auditlist(const char *domain);
 

src/database/message-table.c

@@ -27,7 +27,7 @@
 #include "../gc.h"
 
 static const char *message_types[MAX_MESSAGE] =
-	{ "REGEX", "SUBNET", "HOSTNAME", "DNSMASQ_CONFIG", "RATE_LIMIT", "DNSMASQ_WARN", "LOAD", "SHMEM", "DISK" };
+	{ "REGEX", "SUBNET", "HOSTNAME", "DNSMASQ_CONFIG", "RATE_LIMIT", "DNSMASQ_WARN", "LOAD", "SHMEM", "DISK", "ADLIST" };
 
 static unsigned char message_blob_types[MAX_MESSAGE][5] =
 	{
@@ -94,6 +94,13 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] =
 			SQLITE_NULL, // Not used
 			SQLITE_NULL  // Not used
 		},
+		{	// INACCESSIBLE_ADLIST_MESSAGE: The message column contains the corresponding adlist URL
+			SQLITE_INTEGER, // database index of the adlist (so the dashboard can show a link)
+			SQLITE_NULL, // not used
+			SQLITE_NULL, // not used
+			SQLITE_NULL, // not used
+			SQLITE_NULL // not used
+		},
 	};
 // Create message table in the database
 bool create_message_table(sqlite3 *db)
@@ -391,3 +398,12 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem,
 		add_message(DISK_MESSAGE, true, path, 2, disk, msg);
 	}
 }
+
+void logg_inaccessible_adlist(const int dbindex, const char *address)
+{
+	// Log to FTL.log
+	logg("Adlist warning: Adlist with ID %d (%s) was inaccessible during last gravity run", dbindex, address);
+
+	// Log to database
+	add_message(INACCESSIBLE_ADLIST_MESSAGE, false, address, 1, dbindex);
+}

src/database/message-table.h

@@ -23,5 +23,6 @@ void logg_fatal_dnsmasq_message(const char *message);
 void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit_count);
 void logg_warn_dnsmasq_message(char *message);
 void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg);
+void logg_inaccessible_adlist(const int dbindex, const char *address);
 
 #endif //MESSAGETABLE_H