Improve packagetools regex

1 files changed, 50 insertions, 51 deletions

diff --git a/module/common/packagetools.py b/module/common/packagetools.py
index 5bfbcba95..578148b50 100644
--- a/module/common/packagetools.py
+++ b/module/common/packagetools.py
@@ -1,12 +1,51 @@
-#!/usr/bin/env python
-
-# JDownloader/src/jd/controlling/LinkGrabberPackager.java
+# -*- coding: utf-8 -*-
 
 import re
+
 from urlparse import urlparse
 
+
+endings = ("3gp", "7zip", "7z", "abr", "ac3", "aiff", "aifc", "aif", "ai", "au",
+           "avi", "bin", "bz2", "cbr", "cbz", "ccf", "cue", "cvd", "chm", "dta",
+           "deb", "divx", "djvu", "dlc", "dmg", "doc", "docx", "dot", "eps", "exe",
+           "ff", "flv", "f4v", "gsd", "gif", "gz", "iwd", "iso", "ipsw", "java",
+           "jar", "jpg", "jpeg", "jdeatme", "load", "mws", "mw", "m4v", "m4a",
+           "mkv", "mp2", "mp3", "mp4", "mov", "movie", "mpeg", "mpe", "mpg", "msi",
+           "msu", "msp", "nfo", "npk", "oga", "ogg", "ogv", "otrkey", "pkg", "png",
+           "pdf", "pptx", "ppt", "pps", "ppz", "pot", "psd", "qt", "rmvb", "rm",
+           "rar", "ram", "ra", "rev", "rnd", "r\d+", "rpm", "run", "rsdf", "rtf",
+           "sh(!?tml)", "srt", "snd", "sfv", "swf", "tar", "tif", "tiff", "ts",
+           "txt", "viv", "vivo", "vob", "wav", "wmv", "xla", "xls", "xpi", "zeno",
+           "zip", "z\d+", "_[_a-z]{2}", "\d+$")
+
+rarPats = [re.compile(r'(.*)(\.|_|-)pa?r?t?\.?\d+.(rar|exe)$', re.I),
+           re.compile(r'(.*)(\.|_|-)part\.?[0]*[1].(rar|exe)$', re.I),
+           re.compile(r'(.*)\.rar$', re.I),
+           re.compile(r'(.*)\.r\d+$', re.I),
+           re.compile(r'(.*)(\.|_|-)\d+$', re.I)]
+
+zipPats = [re.compile(r'(.*)\.zip$', re.I),
+           re.compile(r'(.*)\.z\d+$', re.I),
+           re.compile(r'(?is).*\.7z\.[\d]+$', re.I),
+           re.compile(r'(.*)\.a.$', re.I)]
+
+ffsjPats = [re.compile(r'(.*)\._((_[a-z])|([a-z]{2}))(\.|$)'),
+            re.compile(r'(.*)(\.|_|-)[\d]+(\.(' + '|'.join(endings) + ')$)', re.I)]
+
+iszPats = [re.compile(r'(.*)\.isz$', re.I),
+           re.compile(r'(.*)\.i\d{2}$', re.I)]
+
+pat0 = re.compile(r'www\d*\.', re.I)
+
+pat1 = re.compile(r'(\.?CD\d+)', re.I)
+pat2 = re.compile(r'(\.?part\d+)', re.I)
+
+pat3 = re.compile(r'(.+)[\.\-_]+$')
+pat4 = re.compile(r'(.+)\.\d+\.xtm$')
+
+
 def matchFirst(string, *args):
-    """ matches against list of regexp and returns first match"""
+    """ matches against list of regexp and returns first match """
     for patternlist in args:
         for pattern in patternlist:
             r = pattern.search(string)
@@ -21,35 +60,10 @@ def parseNames(files):
     """ Generates packages names from name, data lists
 
     :param files: list of (name, data)
-    :return: packagenames mapt to data lists (eg. urls)
+    :return: packagenames mapped to data lists (eg. urls)
     """
     packs = {}
 
-    endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)"
-
-    rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I),
-               re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I),
-               re.compile("(.*)\\.rar$", re.I),
-               re.compile("(.*)\\.r\\d+$", re.I),
-               re.compile("(.*)(\\.|_|-)\\d+$", re.I)]
-
-    zipPats = [re.compile("(.*)\\.zip$", re.I),
-               re.compile("(.*)\\.z\\d+$", re.I),
-               re.compile("(?is).*\\.7z\\.[\\d]+$", re.I),
-               re.compile("(.*)\\.a.$", re.I)]
-
-    ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"),
-                re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)]
-
-    iszPats = [re.compile("(.*)\\.isz$", re.I),
-               re.compile("(.*)\\.i\\d{2}$", re.I)]
-
-    pat1 = re.compile("(\\.?CD\\d+)", re.I)
-    pat2 = re.compile("(\\.?part\\d+)", re.I)
-
-    pat3 = re.compile("(.+)[\\.\\-_]+$")
-    pat4 = re.compile("(.+)\\.\\d+\\.xtm$")
-
     for file, url in files:
         patternMatch = False
 
@@ -64,7 +78,7 @@ def parseNames(files):
         if len(split) > 1:
             name = split.pop(1)
 
-            #check if a already existing package may be ok for this file
+            #check if an already existing package may be ok for this file
         #        found = False
         #        for pack in packs:
         #            if pack in file:
@@ -72,7 +86,8 @@ def parseNames(files):
         #                found = True
         #                break
         #
-        #        if found: continue
+        #        if found:
+        #            continue
 
         # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern
         before = name
@@ -123,11 +138,12 @@ def parseNames(files):
         # fallback: package by hoster
         if not name:
             name = urlparse(file).hostname
-            if name: name = name.replace("www.", "")
+            if name:
+                name = pat0.sub("", name)
 
         # fallback : default name
         if not name:
-            name = "unknown"
+            name = _("Unnamed package")
 
         # build mapping
         if name in packs:
@@ -136,20 +152,3 @@ def parseNames(files):
             packs[name] = [url]
 
     return packs
-
-
-if __name__ == "__main__":
-    from os.path import join
-    from pprint import pprint
-
-    f = open(join("..", "..", "testlinks2.txt"), "rb")
-    urls = [(x.strip(), x.strip()) for x in f.readlines() if x.strip()]
-    f.close()
-
-    print "Having %d urls." % len(urls)
-
-    packs = parseNames(urls)
-
-    pprint(packs)
-
-    print "Got %d urls." % sum([len(x) for x in packs.itervalues()])