Misc fixes.

scraper/fetchBase.py

@@ -187,7 +187,9 @@ class AbstractFetcher(object, metaclass=abc.ABCMeta):
 		sess = db.session()
 		total_changes = 0
 
-		pbar = tqdm.tqdm(range(self.get_content_count_max(), -1, UPSERT_STEP * -1))
+		item_count = self.get_content_count_max()
+		self.log.info("Max source items = %s", item_count)
+		pbar = tqdm.tqdm(range(item_count + UPSERT_STEP, -1, UPSERT_STEP * -1))
 		for x in pbar:
 
 			# self.log.info("[%s] - Building insert data structure %s -> %s", self.pluginkey, x, x+UPSERT_STEP)
@@ -205,14 +207,13 @@ class AbstractFetcher(object, metaclass=abc.ABCMeta):
 			if changes:
 				sess.commit()
 			pbar.set_description("Changes: %s (%s)" % (changes, total_changes))
-			# if not changes:
-			# 	break
+			if not changes:
+				break
 		self.log.info("[%s] - Done.", self.pluginkey)
 
 
 	def __go(self):
 		self.resetDlstate()
-		self.do_upsert()
 
 		executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.worker_threads)
 		try:

scraper/runner.py

@@ -45,24 +45,24 @@ class RunEngine(object):
 			instance = plugin()
 			instance.do_upsert()
 
-		# threads = []
-		# try:
-		# 	for plugin in PLUGIN_CLASSES:
-		# 		th = threading.Thread(target=plugin.run_scraper, name=plugin.loggerpath)
-		# 		threads.append(th)
+		threads = []
+		try:
+			for plugin in PLUGIN_CLASSES:
+				th = threading.Thread(target=plugin.run_scraper, name=plugin.loggerpath)
+				threads.append(th)
 
 
-		# 	for thread in threads:
-		# 		thread.start()
+			for thread in threads:
+				thread.start()
 
-		# 	self.log.info("Waiting for workers to complete.")
-		# 	for thread in threads:
-		# 		thread.join()
-		# except KeyboardInterrupt:
-		# 	self.log.info("Waiting for executor.")
-		# 	scraper.runstate.run = False
-		# 	for thread in threads:
-		# 		thread.join()
+			self.log.info("Waiting for workers to complete.")
+			for thread in threads:
+				thread.join()
+		except KeyboardInterrupt:
+			self.log.info("Waiting for executor.")
+			scraper.runstate.run = False
+			for thread in threads:
+				thread.join()
 
 def go():
 	instance = RunEngine()