Fixing an issue where the Queue is loaded to full capacity and script blocks indefinitely

Author: Hainish

test/rules/src/https_everywhere_checker/check_rules.py

@@ -83,12 +83,14 @@ def __init__(self, taskQueue, metric, thresholdDistance, autoDisable, resQueue):
 		threading.Thread.__init__(self)
 
 	def run(self):
-		while not self.taskQueue.empty():
+		while True:
 			try:
 				self.processTask(self.taskQueue.get())
 				self.taskQueue.task_done()
 			except Exception, e:
 				logging.exception(e)
+                        if self.taskQueue.empty():
+                            break
 
 	def processTask(self, task):
 		problems = []
@@ -408,6 +410,11 @@ def cli():
 		testedUrlPairCount = 0
 		config.getboolean("debug", "exit_after_dump")
 
+		for i in range(threadCount):
+			t = UrlComparisonThread(taskQueue, metric, thresholdDistance, autoDisable, resQueue)
+			t.setDaemon(True)
+			t.start()
+
 		# set of main pages to test
 		mainPages = set(urlList)
 		# If list of URLs to test/scan was not defined, use the test URL extraction
@@ -426,11 +433,6 @@ def cli():
 				task = ComparisonTask(testUrls, fetcherPlain, fetcher, ruleset)
 				taskQueue.put(task)
 
-		for i in range(threadCount):
-			t = UrlComparisonThread(taskQueue, metric, thresholdDistance, autoDisable, resQueue)
-			t.setDaemon(True)
-			t.start()
-
 		taskQueue.join()
 		logging.info("Finished in %.2f seconds. Loaded rulesets: %d, URL pairs: %d.",
 			time.time() - startTime, len(xmlFnames), testedUrlPairCount)