summaryrefslogtreecommitdiffstats
path: root/module/plugins/internal
diff options
context:
space:
mode:
Diffstat (limited to 'module/plugins/internal')
-rw-r--r--module/plugins/internal/AbstractExtractor.py12
-rw-r--r--module/plugins/internal/MultiHoster.py97
-rw-r--r--module/plugins/internal/NetloadInOCR.py27
-rw-r--r--module/plugins/internal/OCR.py314
-rw-r--r--module/plugins/internal/ShareonlineBizOCR.py53
-rw-r--r--module/plugins/internal/SimpleHoster.py2
-rw-r--r--module/plugins/internal/UnRar.py15
7 files changed, 408 insertions, 112 deletions
diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py
index 2130f910e..3cd635eff 100644
--- a/module/plugins/internal/AbstractExtractor.py
+++ b/module/plugins/internal/AbstractExtractor.py
@@ -13,7 +13,7 @@ class WrongPassword(Exception):
class AbtractExtractor:
@staticmethod
def checkDeps():
- """ Check if system statisfy dependencies
+ """ Check if system satisfies dependencies
:return: boolean
"""
return True
@@ -21,7 +21,7 @@ class AbtractExtractor:
@staticmethod
def getTargets(files_ids):
""" Filter suited targets from list of filename id tuple list
- :param files_ids: List of filepathes
+ :param files_ids: List of file paths
:return: List of targets, id tuple list
"""
raise NotImplementedError
@@ -30,10 +30,10 @@ class AbtractExtractor:
def __init__(self, m, file, out, fullpath, overwrite, renice):
"""Initialize extractor for specific file
- :param m: ExtractArchive Hook plugin
- :param file: Absolute filepath
+ :param m: ExtractArchive addon plugin
+ :param file: Absolute file path
:param out: Absolute path to destination directory
- :param fullpath: extract to fullpath
+ :param fullpath: Extract to fullpath
:param overwrite: Overwrite existing archives
:param renice: Renice value
"""
@@ -52,7 +52,7 @@ class AbtractExtractor:
def checkArchive(self):
- """Check if password if needed. Raise ArchiveError if integrity is
+ """Check if password is needed. Raise ArchiveError if integrity is
questionable.
:return: boolean
diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py
deleted file mode 100644
index e9e321c06..000000000
--- a/module/plugins/internal/MultiHoster.py
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-
-from module.utils import remove_chars
-from module.plugins.Hook import Hook
-
-class MultiHoster(Hook):
- """
- Generic MultiHoster plugin
- """
-
- __version__ = "0.12"
-
- interval = 0
- hosters = []
- replacements = []
- supported = []
- ignored = []
-
- def getHosterCached(self):
- if not self.hosters:
-
- try:
- self.hosters = [x.strip() for x in self.getHoster()]
- self.hosters = filter(lambda x: x and x not in self.ignored, self.hosters)
- except Exception, e:
- self.logError("%s" % str(e))
- return []
-
- for rep in self.replacements:
- if rep[0] in self.hosters:
- self.hosters.remove(rep[0])
- if rep[1] not in self.hosters:
- self.hosters.append(rep[1])
-
- return self.hosters
-
-
- def getHoster(self):
- """Load list of supported hoster
-
- :return: List of domain names
- """
- raise NotImplementedError
-
- def coreReady(self):
- pluginMap = {}
- for name in self.core.pluginManager.hosterPlugins.keys():
- pluginMap[name.lower()] = name
-
- new_supported = []
-
- for hoster in self.getHosterCached():
- name = remove_chars(hoster.lower(), "-.")
-
- if name in pluginMap:
- self.supported.append(pluginMap[name])
- else:
- new_supported.append(hoster)
-
- if not self.supported and not new_supported:
- self.logError(_("No Hoster loaded"))
- return
-
- module = self.core.pluginManager.getPlugin(self.__name__)
- klass = getattr(module, self.__name__)
-
- # inject plugin plugin
- self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported)))
- for hoster in self.supported:
- dict = self.core.pluginManager.hosterPlugins[hoster]
- dict["new_module"] = module
- dict["new_name"] = self.__name__
-
- self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported)))
-
- # create new regexp
- if not klass.__pattern__:
- regexp = r".*(%s).*" % "|".join([x.replace(".", "\\.") for x in new_supported])
- else:
- regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported])
-
- dict = self.core.pluginManager.hosterPlugins[self.__name__]
- dict["pattern"] = regexp
- dict["re"] = re.compile(regexp)
-
-
- def unload(self):
- for hoster in self.supported:
- dict = self.core.pluginManager.hosterPlugins[hoster]
- if "module" in dict:
- del dict["module"]
-
- del dict["new_module"]
- del dict["new_name"]
diff --git a/module/plugins/internal/NetloadInOCR.py b/module/plugins/internal/NetloadInOCR.py
new file mode 100644
index 000000000..e50978701
--- /dev/null
+++ b/module/plugins/internal/NetloadInOCR.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+from OCR import OCR
+
+class NetloadInOCR(OCR):
+ __version__ = 0.1
+
+ def __init__(self):
+ OCR.__init__(self)
+
+ def get_captcha(self, image):
+ self.load_image(image)
+ self.to_greyscale()
+ self.clean(3)
+ self.clean(3)
+ self.run_tesser(True, True, False, False)
+
+ self.result_captcha = self.result_captcha.replace(" ", "")[:4] # cut to 4 numbers
+
+ return self.result_captcha
+
+if __name__ == '__main__':
+ import urllib
+ ocr = NetloadInOCR()
+ urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png")
+
+ print ocr.get_captcha('captcha.png')
diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py
new file mode 100644
index 000000000..9f8b7ef8c
--- /dev/null
+++ b/module/plugins/internal/OCR.py
@@ -0,0 +1,314 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#Copyright (C) 2009 kingzero, RaNaN
+#
+#This program is free software; you can redistribute it and/or modify
+#it under the terms of the GNU General Public License as published by
+#the Free Software Foundation; either version 3 of the License,
+#or (at your option) any later version.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#See the GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+#
+###
+from __future__ import with_statement
+import os
+from os.path import join
+from os.path import abspath
+import logging
+import subprocess
+#import tempfile
+
+import Image
+import TiffImagePlugin
+import PngImagePlugin
+import GifImagePlugin
+import JpegImagePlugin
+
+
+class OCR(object):
+ __version__ = 0.1
+
+ def __init__(self):
+ self.logger = logging.getLogger("log")
+
+ def load_image(self, image):
+ self.image = Image.open(image)
+ self.pixels = self.image.load()
+ self.result_captcha = ''
+
+ def unload(self):
+ """delete all tmp images"""
+ pass
+
+ def threshold(self, value):
+ self.image = self.image.point(lambda a: a * value + 10)
+
+ def run(self, command):
+ """Run a command"""
+
+ popen = subprocess.Popen(command, bufsize = -1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ popen.wait()
+ output = popen.stdout.read() +" | "+ popen.stderr.read()
+ popen.stdout.close()
+ popen.stderr.close()
+ self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output))
+
+ def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True):
+ #self.logger.debug("create tmp tif")
+
+
+ #tmp = tempfile.NamedTemporaryFile(suffix=".tif")
+ tmp = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb")
+ tmp.close()
+ #self.logger.debug("create tmp txt")
+ #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt")
+ tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb")
+ tmpTxt.close()
+
+ self.logger.debug("save tiff")
+ self.image.save(tmp.name, 'TIFF')
+
+ if os.name == "nt":
+ tessparams = [join(pypath,"tesseract","tesseract.exe")]
+ else:
+ tessparams = ['tesseract']
+
+ tessparams.extend( [abspath(tmp.name), abspath(tmpTxt.name).replace(".txt", "")] )
+
+ if subset and (digits or lowercase or uppercase):
+ #self.logger.debug("create temp subset config")
+ #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset")
+ tmpSub = open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb")
+ tmpSub.write("tessedit_char_whitelist ")
+ if digits:
+ tmpSub.write("0123456789")
+ if lowercase:
+ tmpSub.write("abcdefghijklmnopqrstuvwxyz")
+ if uppercase:
+ tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+ tmpSub.write("\n")
+ tessparams.append("nobatch")
+ tessparams.append(abspath(tmpSub.name))
+ tmpSub.close()
+
+ self.logger.debug("run tesseract")
+ self.run(tessparams)
+ self.logger.debug("read txt")
+
+ try:
+ with open(tmpTxt.name, 'r') as f:
+ self.result_captcha = f.read().replace("\n", "")
+ except:
+ self.result_captcha = ""
+
+ self.logger.debug(self.result_captcha)
+ try:
+ os.remove(tmp.name)
+ os.remove(tmpTxt.name)
+ if subset and (digits or lowercase or uppercase):
+ os.remove(tmpSub.name)
+ except:
+ pass
+
+ def get_captcha(self, name):
+ raise NotImplementedError
+
+ def to_greyscale(self):
+ if self.image.mode != 'L':
+ self.image = self.image.convert('L')
+
+ self.pixels = self.image.load()
+
+ def eval_black_white(self, limit):
+ self.pixels = self.image.load()
+ w, h = self.image.size
+ for x in xrange(w):
+ for y in xrange(h):
+ if self.pixels[x, y] > limit:
+ self.pixels[x, y] = 255
+ else:
+ self.pixels[x, y] = 0
+
+ def clean(self, allowed):
+ pixels = self.pixels
+
+ w, h = self.image.size
+
+ for x in xrange(w):
+ for y in xrange(h):
+ if pixels[x, y] == 255: continue
+ # no point in processing white pixels since we only want to remove black pixel
+ count = 0
+
+ try:
+ if pixels[x-1, y-1] != 255: count += 1
+ if pixels[x-1, y] != 255: count += 1
+ if pixels[x-1, y + 1] != 255: count += 1
+ if pixels[x, y + 1] != 255: count += 1
+ if pixels[x + 1, y + 1] != 255: count += 1
+ if pixels[x + 1, y] != 255: count += 1
+ if pixels[x + 1, y-1] != 255: count += 1
+ if pixels[x, y-1] != 255: count += 1
+ except:
+ pass
+
+ # not enough neighbors are dark pixels so mark this pixel
+ # to be changed to white
+ if count < allowed:
+ pixels[x, y] = 1
+
+ # second pass: this time set all 1's to 255 (white)
+ for x in xrange(w):
+ for y in xrange(h):
+ if pixels[x, y] == 1: pixels[x, y] = 255
+
+ self.pixels = pixels
+
+ def derotate_by_average(self):
+ """rotate by checking each angle and guess most suitable"""
+
+ w, h = self.image.size
+ pixels = self.pixels
+
+ for x in xrange(w):
+ for y in xrange(h):
+ if pixels[x, y] == 0:
+ pixels[x, y] = 155
+
+ highest = {}
+ counts = {}
+
+ for angle in range(-45, 45):
+
+ tmpimage = self.image.rotate(angle)
+
+ pixels = tmpimage.load()
+
+ w, h = self.image.size
+
+ for x in xrange(w):
+ for y in xrange(h):
+ if pixels[x, y] == 0:
+ pixels[x, y] = 255
+
+
+ count = {}
+
+ for x in xrange(w):
+ count[x] = 0
+ for y in xrange(h):
+ if pixels[x, y] == 155:
+ count[x] += 1
+
+ sum = 0
+ cnt = 0
+
+ for x in count.values():
+ if x != 0:
+ sum += x
+ cnt += 1
+
+ avg = sum / cnt
+ counts[angle] = cnt
+ highest[angle] = 0
+ for x in count.values():
+ if x > highest[angle]:
+ highest[angle] = x
+
+ highest[angle] = highest[angle] - avg
+
+ hkey = 0
+ hvalue = 0
+
+ for key, value in highest.iteritems():
+ if value > hvalue:
+ hkey = key
+ hvalue = value
+
+ self.image = self.image.rotate(hkey)
+ pixels = self.image.load()
+
+ for x in xrange(w):
+ for y in xrange(h):
+ if pixels[x, y] == 0:
+ pixels[x, y] = 255
+
+ if pixels[x, y] == 155:
+ pixels[x, y] = 0
+
+ self.pixels = pixels
+
+ def split_captcha_letters(self):
+ captcha = self.image
+ started = False
+ letters = []
+ width, height = captcha.size
+ bottomY, topY = 0, height
+ pixels = captcha.load()
+
+ for x in xrange(width):
+ black_pixel_in_col = False
+ for y in xrange(height):
+ if pixels[x, y] != 255:
+ if not started:
+ started = True
+ firstX = x
+ lastX = x
+
+ if y > bottomY: bottomY = y
+ if y < topY: topY = y
+ if x > lastX: lastX = x
+
+ black_pixel_in_col = True
+
+ if black_pixel_in_col == False and started == True:
+ rect = (firstX, topY, lastX, bottomY)
+ new_captcha = captcha.crop(rect)
+
+ w, h = new_captcha.size
+ if w > 5 and h > 5:
+ letters.append(new_captcha)
+
+ started = False
+ bottomY, topY = 0, height
+
+ return letters
+
+ def correct(self, values, var=None):
+
+ if var:
+ result = var
+ else:
+ result = self.result_captcha
+
+ for key, item in values.iteritems():
+
+ if key.__class__ == str:
+ result = result.replace(key, item)
+ else:
+ for expr in key:
+ result = result.replace(expr, item)
+
+ if var:
+ return result
+ else:
+ self.result_captcha = result
+
+
+if __name__ == '__main__':
+ ocr = OCR()
+ ocr.load_image("B.jpg")
+ ocr.to_greyscale()
+ ocr.eval_black_white(140)
+ ocr.derotate_by_average()
+ ocr.run_tesser()
+ print "Tesseract", ocr.result_captcha
+ ocr.image.save("derotated.jpg")
+
diff --git a/module/plugins/internal/ShareonlineBizOCR.py b/module/plugins/internal/ShareonlineBizOCR.py
new file mode 100644
index 000000000..c5c2e92e8
--- /dev/null
+++ b/module/plugins/internal/ShareonlineBizOCR.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#Copyright (C) 2009 kingzero, RaNaN
+#
+#This program is free software; you can redistribute it and/or modify
+#it under the terms of the GNU General Public License as published by
+#the Free Software Foundation; either version 3 of the License,
+#or (at your option) any later version.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#See the GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+#
+###
+from OCR import OCR
+
+class ShareonlineBizOCR(OCR):
+ __version__ = 0.1
+
+ def __init__(self):
+ OCR.__init__(self)
+
+ def get_captcha(self, image):
+ self.load_image(image)
+ self.to_greyscale()
+ self.image = self.image.resize((160, 50))
+ self.pixels = self.image.load()
+ self.threshold(1.85)
+ #self.eval_black_white(240)
+ #self.derotate_by_average()
+
+ letters = self.split_captcha_letters()
+
+ final = ""
+ for letter in letters:
+ self.image = letter
+ self.run_tesser(True, True, False, False)
+ final += self.result_captcha
+
+ return final
+
+ #tesseract at 60%
+
+if __name__ == '__main__':
+ import urllib
+ ocr = ShareonlineBizOCR()
+ urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg")
+ print ocr.get_captcha('captcha.jpeg')
diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py
index 6d7bb6284..a2e246d44 100644
--- a/module/plugins/internal/SimpleHoster.py
+++ b/module/plugins/internal/SimpleHoster.py
@@ -141,7 +141,7 @@ class SimpleHoster(Hoster):
or FILE_NAME_INFO = r'(?P<N>file_name)'
and FILE_SIZE_INFO = r'(?P<S>file_size) (?P<U>units)'
FILE_OFFLINE_PATTERN = r'File (deleted|not found)'
- TEMP_OFFLINE_PATTERN = r'Server maintainance'
+ TEMP_OFFLINE_PATTERN = r'Server maintenance'
"""
FILE_SIZE_REPLACEMENTS = []
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 240dc0233..53995a083 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -19,11 +19,10 @@
import os
import re
-from os.path import join
from glob import glob
from subprocess import Popen, PIPE
-from module.utils import save_join, decode
+from module.utils.fs import save_join, decode, fs_encode
from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError
class UnRar(AbtractExtractor):
@@ -39,7 +38,7 @@ class UnRar(AbtractExtractor):
@staticmethod
def checkDeps():
if os.name == "nt":
- UnRar.CMD = join(pypath, "UnRAR.exe")
+ UnRar.CMD = save_join(pypath, "UnRAR.exe")
p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE)
p.communicate()
else:
@@ -80,7 +79,7 @@ class UnRar(AbtractExtractor):
self.password = "" #save the correct password
def checkArchive(self):
- p = self.call_unrar("l", "-v", self.file)
+ p = self.call_unrar("l", "-v", fs_encode(self.file))
out, err = p.communicate()
if self.re_wrongpwd.search(err):
self.passwordProtected = True
@@ -102,7 +101,7 @@ class UnRar(AbtractExtractor):
def checkPassword(self, password):
#at this point we can only verify header protected files
if self.headerProtected:
- p = self.call_unrar("l", "-v", self.file, password=password)
+ p = self.call_unrar("l", "-v", fs_encode(self.file), password=password)
out, err = p.communicate()
if self.re_wrongpwd.search(err):
return False
@@ -115,7 +114,7 @@ class UnRar(AbtractExtractor):
# popen thinks process is still alive (just like pexpect) - very strange behavior
# so for now progress can not be determined correctly
- p = self.call_unrar(command, self.file, self.out, password=password)
+ p = self.call_unrar(command, fs_encode(self.file), self.out, password=password)
renice(p.pid, self.renice)
progress(0)
@@ -143,7 +142,7 @@ class UnRar(AbtractExtractor):
def listContent(self):
command = "vb" if self.fullpath else "lb"
- p = self.call_unrar(command, "-v", self.file, password=self.password)
+ p = self.call_unrar(command, "-v", fs_encode(self.file), password=self.password)
out, err = p.communicate()
if "Cannot open" in err:
@@ -178,7 +177,7 @@ class UnRar(AbtractExtractor):
#NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
call = [self.CMD, command] + args + list(xargs)
- self.m.logDebug(" ".join(call))
+ self.m.logDebug(" ".join([decode(arg) for arg in call]))
p = Popen(call, stdout=PIPE, stderr=PIPE)
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-24 19:33:56 +0100