updated thrift, --clean method, youtube fix closed #379

11 files changed, 74 insertions, 255 deletions

diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py
index d93fa6da4..19205dac6 100644
--- a/module/database/FileDatabase.py
+++ b/module/database/FileDatabase.py
@@ -750,7 +750,7 @@ class FileMethods():
     @style.queue    
     def updateLinkInfo(self, data):
         """ data is list of tupels (name, size, status, url) """
-        self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status NOT IN (0,8,12,13)', data)
+        self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data)
         ids = []
         self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data]))
         for r in self.c:
diff --git a/module/lib/thrift/Thrift.py b/module/lib/thrift/Thrift.py
index a96351276..1d271fcff 100644
--- a/module/lib/thrift/Thrift.py
+++ b/module/lib/thrift/Thrift.py
@@ -96,6 +96,8 @@ class TApplicationException(TException):
   WRONG_METHOD_NAME = 3
   BAD_SEQUENCE_ID = 4
   MISSING_RESULT = 5
+  INTERNAL_ERROR = 6
+  PROTOCOL_ERROR = 7
 
   def __init__(self, type=UNKNOWN, message=None):
     TException.__init__(self, message)
diff --git a/module/lib/thrift/protocol/TBase.py b/module/lib/thrift/protocol/TBase.py
index dfe0d79ce..e675c7dc0 100644
--- a/module/lib/thrift/protocol/TBase.py
+++ b/module/lib/thrift/protocol/TBase.py
@@ -69,230 +69,4 @@ class TExceptionBase(Exception):
   __ne__ = TBase.__ne__.im_func
   read = TBase.read.im_func
   write = TBase.write.im_func
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-from thrift.Thrift import *
-from thrift.protocol import TBinaryProtocol
-from thrift.transport import TTransport
-
-try:
-  from thrift.protocol import fastbinary
-except:
-  fastbinary = None
-
-def read(iprot, types, ftype, spec):
-    try:
-        return types[ftype][0]()
-    except KeyError:
-        if ftype == TType.LIST:
-            ltype, lsize = iprot.readListBegin()
-
-            value = [read(iprot, types, spec[0], spec[1]) for i in range(lsize)]
-
-            iprot.readListEnd()
-            return value
-        
-        elif ftype == TType.SET:
-            ltype, lsize = iprot.readSetBegin()
-
-            value = set([read(iprot, types, spec[0], spec[1]) for i in range(lsize)])
-
-            iprot.readSetEnd()
-            return value
-
-        elif ftype == TType.MAP:
-            key_type, key_spec = spec[0], spec[1]
-            val_type, val_spec = spec[2], spec[3]
-
-            ktype, vtype, mlen = iprot.readMapBegin()
-            res = dict()
-
-            for i in xrange(mlen):
-                key = read(iprot, types, key_type, key_spec)
-                res[key] = read(iprot, types, val_type, val_spec)
-
-            iprot.readMapEnd()
-            return res
-
-        elif ftype == TType.STRUCT:
-            return spec[0]().read(iprot)
-
-
-
-
-def write(oprot, types, ftype, spec, value):
-    try:
-        types[ftype][1](value)
-    except KeyError:
-        if ftype == TType.LIST:
-            oprot.writeListBegin(spec[0], len(value))
-
-            for elem in value:
-                write(oprot, types, spec[0], spec[1], elem)
-
-            oprot.writeListEnd()
-        elif ftype == TType.SET:
-            oprot.writeSetBegin(spec[0], len(value))
-
-            for elem in value:
-                write(oprot, types, spec[0], spec[1], elem)
-                
-            oprot.writeSetEnd()
-        elif ftype == TType.MAP:
-            key_type, key_spec = spec[0], spec[1]
-            val_type, val_spec = spec[2], spec[3]
-
-            oprot.writeMapBegin(key_type, val_type, len(value))
-            for key, val in value.iteritems():
-                write(oprot, types, key_type, key_spec, key)
-                write(oprot, types, val_type, val_spec, val)
-
-            oprot.writeMapEnd()
-        elif ftype == TType.STRUCT:
-            value.write(oprot)
-
-
-class TBase2(object):
-    __slots__ = ("thrift_spec")
-
-    #subclasses provides this information
-    thrift_spec = ()
-
-    def __repr__(self):
-        L = ['%s=%r' % (key, getattr(self, key))
-              for key in self.__slots__ ]
-        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
-
-    def __eq__(self, other):
-        if not isinstance(other, self.__class__):
-            return False
-        for attr in self.__slots__:
-            my_val = getattr(self, attr)
-            other_val = getattr(other, attr)
-            if my_val != other_val:
-                return False
-        return True
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def read(self, iprot):
-        if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
-            fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
-            return
-
-        #local copies for faster access
-        thrift_spec = self.thrift_spec
-        setter = self.__setattr__
-
-        iprot.readStructBegin()
-        while True:
-            (fname, ftype, fid) = iprot.readFieldBegin()
-            if ftype == TType.STOP:
-                break
-
-            try:
-                specs = thrift_spec[fid]
-                if not specs or specs[1] != ftype:
-                    iprot.skip(ftype)
-
-                else:
-                    pos, etype, ename, espec, unk = specs
-                    value = read(iprot, iprot.primTypes, etype, espec)
-                    setter(ename, value)
-                    
-            except IndexError:
-                iprot.skip()
-
-            iprot.readFieldEnd()
-
-        iprot.readStructEnd()
-
-    def write(self, oprot):
-        if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
-            oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
-            return
-
-        #local copies for faster access
-        oprot.writeStructBegin(self.__class__.__name__)
-        getter = self.__getattribute__
-
-        for spec in self.thrift_spec:
-            if spec is None: continue
-            # element attributes
-            pos, etype, ename, espec, unk = spec
-            value = getter(ename)
-            if value is None: continue
-
-            oprot.writeFieldBegin(ename, etype, pos)
-            write(oprot, oprot.primTypes, etype, espec, value)
-            oprot.writeFieldEnd()
-
-        oprot.writeFieldStop()
-        oprot.writeStructEnd()
-
-class TBase(object):
-  __slots__ = ('thrift_spec',)
-
-  #provides by subclasses
-  thrift_spec = ()
-
-  def __repr__(self):
-    L = ['%s=%r' % (key, getattr(self, key))
-              for key in self.__slots__ ]
-    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
-
-  def __eq__(self, other):
-    if not isinstance(other, self.__class__):
-      return False
-    for attr in self.__slots__:
-      my_val = getattr(self, attr)
-      other_val = getattr(other, attr)
-      if my_val != other_val:
-        return False
-    return True
-    
-  def __ne__(self, other):
-    return not (self == other)
   
-  def read(self, iprot):
-    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
-      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
-      return
-    iprot.readStruct(self, self.thrift_spec)
-
-  def write(self, oprot):
-    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
-      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
-      return
-    oprot.writeStruct(self, self.thrift_spec)
-
-class TExceptionBase(Exception):
-  # old style class so python2.4 can raise exceptions derived from this
-  #  This can't inherit from TBase because of that limitation.
-  __slots__ = []
-  
-  __repr__ = TBase.__repr__.im_func
-  __eq__ = TBase.__eq__.im_func
-  __ne__ = TBase.__ne__.im_func
-  read = TBase.read.im_func
-  write = TBase.write.im_func
-
diff --git a/module/lib/thrift/protocol/TCompactProtocol.py b/module/lib/thrift/protocol/TCompactProtocol.py
index 280b54f0f..016a33171 100644
--- a/module/lib/thrift/protocol/TCompactProtocol.py
+++ b/module/lib/thrift/protocol/TCompactProtocol.py
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 from TProtocol import *
 from struct import pack, unpack
 
@@ -204,7 +223,10 @@ class TCompactProtocol(TProtocolBase):
             ctype = CompactType.FALSE
         self.__writeFieldHeader(ctype, self.__bool_fid)
     elif self.state == CONTAINER_WRITE:
-      self.__writeByte(int(bool))
+       if bool:
+           self.__writeByte(CompactType.TRUE)
+       else:
+           self.__writeByte(CompactType.FALSE)
     else:
       raise AssertionError, "Invalid state in compact protocol"
 
@@ -338,9 +360,9 @@ class TCompactProtocol(TProtocolBase):
 
   def readBool(self):
     if self.state == BOOL_READ:
-      return self.__bool_value
+      return self.__bool_value == CompactType.TRUE
     elif self.state == CONTAINER_READ:
-      return bool(self.__readByte())
+      return self.__readByte() == CompactType.TRUE
     else:
       raise AssertionError, "Invalid state in compact protocol: %d" % self.state
 
diff --git a/module/lib/thrift/protocol/TProtocol.py b/module/lib/thrift/protocol/TProtocol.py
index beb6bea16..7338ff68a 100644
--- a/module/lib/thrift/protocol/TProtocol.py
+++ b/module/lib/thrift/protocol/TProtocol.py
@@ -247,10 +247,11 @@ class TProtocolBase:
       for idx in xrange(list_len):
         results.append(reader())
     else:
-      (elem_class, elem_spec) = tspec
+      # this is like an inlined readFieldByTType
+      container_reader = self._TTYPE_HANDLERS[list_type][0]
+      val_reader = getattr(self, container_reader)
       for idx in xrange(list_len):
-        val = elem_class()
-        val.read(self)
+        val = val_reader(tspec)
         results.append(val)
     self.readListEnd()
     return results
@@ -260,17 +261,16 @@ class TProtocolBase:
     ttype, tspec = spec[0], spec[1]
     r_handler = self._TTYPE_HANDLERS[ttype][0]
     reader = getattr(self, r_handler)
-    (list_type, set_len) = self.readSetBegin()
+    (set_type, set_len) = self.readSetBegin()
     if tspec is None:
-      # list values are simple types
+      # set members are simple types
       for idx in xrange(set_len):
         results.add(reader())
     else:
-      (elem_class, elem_spec) = tspec
+      container_reader = self._TTYPE_HANDLERS[set_type][0]
+      val_reader = getattr(self, container_reader)
       for idx in xrange(set_len):
-        val = elem_class()
-        val.read(self)
-        results.add(val)
+        results.add(val_reader(tspec)) 
     self.readSetEnd()
     return results
 
@@ -336,7 +336,7 @@ class TProtocolBase:
         e_writer(elem)
     else:
       for elem in val:
-        e_writer(elem, spec)
+        e_writer(elem, spec[1])
     self.writeListEnd()
 
   def writeContainerSet(self, val, spec):
@@ -348,7 +348,7 @@ class TProtocolBase:
         e_writer(elem)
     else:
       for elem in val:
-        e_writer(elem, spec)
+        e_writer(elem, spec[1])
     self.writeSetEnd()
 
   def writeContainerMap(self, val, spec):
diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py
index f02124a8a..f5c078471 100644
--- a/module/plugins/hoster/YoutubeCom.py
+++ b/module/plugins/hoster/YoutubeCom.py
@@ -11,13 +11,13 @@ class YoutubeCom(Hoster):
     __name__ = "YoutubeCom"
     __type__ = "hoster"
     __pattern__ = r"http://(www\.)?(de\.)?\youtube\.com/watch\?v=.*"
-    __version__ = "0.22"
+    __version__ = "0.23"
     __config__ = [("quality", "sd;hd;fullhd", "Quality Setting", "hd"),
-            ("fmt", "int", "FMT Number 0-45", 0),
-            (".mp4", "bool", "Allow .mp4", True),
-            (".flv", "bool", "Allow .flv", True),
-            (".webm", "bool", "Allow .webm", False),
-            (".3gp", "bool", "Allow .3gp", False)]
+        ("fmt", "int", "FMT Number 0-45", 0),
+        (".mp4", "bool", "Allow .mp4", True),
+        (".flv", "bool", "Allow .flv", True),
+        (".webm", "bool", "Allow .webm", False),
+        (".3gp", "bool", "Allow .3gp", False)]
     __description__ = """Youtube.com Video Download Hoster"""
     __author_name__ = ("spoob")
     __author_mail__ = ("spoob@pyload.org")
@@ -43,6 +43,9 @@ class YoutubeCom(Hoster):
         if "watch-player-unavailable" in html:
             self.offline()
 
+        if "We have been receiving a large volume of requests from your network." in html:
+            self.tempOffline()
+
         #videoId = pyfile.url.split("v=")[1].split("&")[0]
         #videoHash = re.search(r'&t=(.+?)&', html).group(1)
 
@@ -63,9 +66,8 @@ class YoutubeCom(Hoster):
 
         flashvars = re.search(r"flashvars=\"([^\"]+)", html)
         flashvars = unquote(flashvars.group(1))
-        
-        fmts =  re.findall(r"itag=(\d+),url=([^&]+)", flashvars)
 
+        fmts = re.findall(r"itag=(\d+),url=([^&]+)", flashvars)
 
         fmt_dict = {}
         for fmt, url in fmts:
diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
index fd48556dd..6c8116d92 100755
--- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
+++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Autogenerated by Thrift Compiler (0.7.0-dev)
+# Autogenerated by Thrift Compiler (0.8.0-dev)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
index ef2b037d5..a29eed62f 100644
--- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
+++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
@@ -1,5 +1,5 @@
 #
-# Autogenerated by Thrift Compiler (0.7.0-dev)
+# Autogenerated by Thrift Compiler (0.8.0-dev)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
diff --git a/module/remote/thriftbackend/thriftgen/pyload/constants.py b/module/remote/thriftbackend/thriftgen/pyload/constants.py
index 7c0918a6d..1ca9842ff 100644
--- a/module/remote/thriftbackend/thriftgen/pyload/constants.py
+++ b/module/remote/thriftbackend/thriftgen/pyload/constants.py
@@ -1,5 +1,5 @@
 #
-# Autogenerated by Thrift Compiler (0.7.0-dev)
+# Autogenerated by Thrift Compiler (0.8.0-dev)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
diff --git a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
index 4ff110e58..8b0428abd 100644
--- a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
+++ b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
@@ -1,5 +1,5 @@
 #
-# Autogenerated by Thrift Compiler (0.7.0-dev)
+# Autogenerated by Thrift Compiler (0.8.0-dev)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
diff --git a/pyLoadCore.py b/pyLoadCore.py
index c733c943c..be916f712 100755
--- a/pyLoadCore.py
+++ b/pyLoadCore.py
@@ -30,7 +30,7 @@ from imp import find_module
 import logging
 import logging.handlers
 import os
-from os import _exit, execv, getcwd, makedirs, remove, sep
+from os import _exit, execv, getcwd, makedirs, remove, sep, walk
 from os.path import exists, join
 import signal
 import subprocess
@@ -65,6 +65,8 @@ else:
 
 sys.stdout = getwriter(enc)(sys.stdout, errors = "replace")
 
+# TODO List
+
 class Core(object):
     """pyLoad Core, one tool to rule them all... (the filehosters) :D"""
 
@@ -79,7 +81,7 @@ class Core(object):
         if len(argv) > 1:
             try:
                 options, args = getopt(argv[1:], 'vchdusq',
-                                       ["version", "clear", "help", "debug", "user", "setup", "configdir=", "changedir", "daemon", "quit", "status"])
+                                       ["version", "clear", "clean", "help", "debug", "user", "setup", "configdir=", "changedir", "daemon", "quit", "status"])
 
                 for option, argument in options:
                     if option in ("-v", "--version"):
@@ -125,6 +127,9 @@ class Core(object):
                     elif option == "--status":
                         print self.isAlreadyRunning()
                         exit()
+                    elif option == "--clean":
+                        self.cleanTree()
+                        exit()
                         
             except GetoptError:
                 print 'Unknown Argument(s) "%s"' % " ".join(argv[1:])
@@ -151,6 +156,7 @@ class Core(object):
         print "  --changedir", " "* 12, "Change config dir permanently"
         print "  --daemon", " " * 15, "Daemonmize after start"
         print "  --status", " " * 15, "Display pid if running or False"
+        print "  --clean", " " * 16 , "Remove .pyc/.pyo files"
         print "  -q, --quit", " " * 13, "Quit running pyLoad instance"
         print "  -h, --help", " " * 13, "Display this help screen"
         print ""
@@ -230,6 +236,19 @@ class Core(object):
 
         except:
             print "Error quitting pyLoad"
+
+
+    def cleanTree(self):
+        for path, dirs, files in walk(self.path("")):
+            for f in files:
+                if not f.endswith(".pyo") and not f.endswith(".pyc"):
+                    continue
+
+                if "_25" in f or "_26" in f or "_27" in f:
+                    continue
+
+                print join(path, f)
+                remove(join(path, f))
     
     def start(self, rpc=True, web=True):
         """ starts the fun :D """