More updates, some minor patches to handle missing images on gelbooru.

README.md

@@ -13,4 +13,8 @@ Potential ideas:
  - Add support for:
 	 - http://www.zerochan.net/
 	 - http://e-shuushuu.net/
-	 - https://konachan.com/
+	 - https://konachan.com/
+	 - https://chan.sankakucomplex.com/
+	 - http://rule34.xxx/
+	 - http://paheal.net/
+	 - https://e621.net/ (furry?)

danbooruFetch.py

@@ -22,6 +22,9 @@ class DanbooruFetcher(fetchBase.AbstractFetcher):
 	pluginkey = 'Danbooru'
 	loggerpath = "Main.Danbooru"
 
+	def __init__(self):
+		self.log = logging.getLogger("Main.Danbooru")
+		self.wg = webFunctions.WebGetRobust(logPath="Main.Danbooru.Web")
 
 	def extractTags(self, job, tagsection):
 
@@ -147,6 +150,10 @@ class DanbooruFetcher(fetchBase.AbstractFetcher):
 			job.dlstate=-4
 			db.session.commit()
 			return
+		if 'Save this flash' in text:
+			job.dlstate=-9
+			db.session.commit()
+			return
 		err = 0
 		while err < 5:
 			try:

fetchBase.py

@@ -35,7 +35,7 @@ class AbstractFetcher(object, metaclass=abc.ABCMeta):
 
 	def __init__(self):
 		self.log = logging.getLogger(self.loggerpath)
-		self.wg = webFunctions.WebGetRobust()
+
 
 		# db.session = db.Session()
 

gelbooruFetch.py

@@ -18,7 +18,7 @@ import datetime
 class GelbooruFetcher(object):
 	def __init__(self):
 		self.log = logging.getLogger("Main.Gelbooru")
-		self.wg = webFunctions.WebGetRobust()
+		self.wg = webFunctions.WebGetRobust(logPath="Main.Gelbooru.Web")
 
 		# db.session = db.Session()
 
@@ -103,6 +103,7 @@ class GelbooruFetcher(object):
 			elif name == 'Posted':
 				cal = parsedatetime.Calendar()
 				itemdate = val.split("at")[0]
+				itemdate = val.split("by")[0]
 				tstruct, pstat = cal.parse(itemdate)
 				assert pstat == 1 or pstat == 2
 				job.posted = datetime.datetime.fromtimestamp(time.mktime(tstruct))
@@ -219,6 +220,12 @@ class GelbooruFetcher(object):
 			db.session.commit()
 			return
 
+		if 'This post was deleted. Reason: Duplicate of' in soup.get_text():
+			self.log.warn("Image has been removed.")
+			job.dlstate=-6
+			db.session.commit()
+			return
+
 		# text = soup.get_text()
 		# if 'You need a gold account to see this image.' in text:
 		# 	job.dlstate=-3
@@ -238,6 +245,9 @@ class GelbooruFetcher(object):
 			except sqlalchemy.exc.IntegrityError:
 				err += 1
 				db.session.rollback()
+			except urllib.error.URLError:
+				job.dlstate=-8
+				db.session.commit()
 
 
 

logSetup.py

@@ -36,10 +36,10 @@ class ColourHandler(logging.Handler):
 
 		segments = record.name.split(".")
 		tname = threading.current_thread().name
+		segments.append(tname)
 		if segments[0] == "Main" and len(segments) > 1:
-			# segments.pop(0)
-			# segments[0] = "Main."+segments[0]
-			segments[0] = "Main."+tname
+			segments.pop(0)
+			segments[0] = "Main."+segments[0]
 
 		nameList = []
 

main.py

@@ -9,7 +9,7 @@ import gelbooruFetch
 import runstate
 import concurrent.futures
 
-THREADS = 1
+# THREADS = 6
 THREADS = 30
 
 def insertDanbooruStartingPoints():
@@ -72,6 +72,8 @@ def go():
 	executor = concurrent.futures.ThreadPoolExecutor(max_workers=THREADS)
 	try:
 		# for x in range(2):
+		# executor.submit(danbooruFetch.run, 0)
+		# executor.submit(gelbooruFetch.run, 0)
 		for x in range(THREADS//2):
 			executor.submit(danbooruFetch.run, x)
 		for x in range(THREADS//2):