diff options
Diffstat (limited to 'module')
-rw-r--r-- | module/download_thread.py | 19 | ||||
-rw-r--r-- | module/file_list.py | 45 | ||||
-rw-r--r-- | module/thread_list.py | 60 |
3 files changed, 49 insertions, 75 deletions
diff --git a/module/download_thread.py b/module/download_thread.py index a2318038a..a290bc4f7 100644 --- a/module/download_thread.py +++ b/module/download_thread.py @@ -60,16 +60,15 @@ class Download_Thread(threading.Thread): def run(self): while (not self.shutdown): - if self.parent.py_load_files: - self.loadedPyFile = self.parent.get_job() - if self.loadedPyFile: - try: - self.download(self.loadedPyFile) - except Exception, e: - print "Error:", e #catch up all error here - self.loadedPyFile.status.type = "failed" - finally: - self.parent.job_finished(self.loadedPyFile) + self.loadedPyFile = self.parent.get_job() + if self.loadedPyFile: + try: + self.download(self.loadedPyFile) + except Exception, e: + print "Error:", e #catch up all error here + self.loadedPyFile.status.type = "failed" + finally: + self.parent.job_finished(self.loadedPyFile) sleep(0.5) if self.shutdown: sleep(1) diff --git a/module/file_list.py b/module/file_list.py index e8cc49ab2..c9a96635f 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -22,18 +22,18 @@ LIST_VERSION = 1 import cPickle from Py_Load_File import PyLoadFile - + class File_List(object): - def __init__(self): - self.version = LIST_VERSION + def __init__(self, core): + self.core = core self.files = [] + self.data = {'version': LIST_VERSION} self.id = 0 + #self.load() - def set_core(self, core): - self.core = core - - def new_pyfile(self): - pyfile = PyLoadFile(self, url) + def new_pyfile(self, url): + url = url.replace("\n", "") + pyfile = PyLoadFile(self.core, url) pyfile.download_folder = self.core.config['download_folder'] pyfile.id = self.id self.id += 1 @@ -41,23 +41,26 @@ class File_List(object): return pyfile def append(self, url): - self.links.append(url) + self.files.append(self.new_pyfile(url)) + + def extend(self, urls): + for url in urls: + self.append(url) def save(self): output = open('links.pkl', 'wb') - cPickle.dump(self, output, -1) - + cPickle.dump(self.data, output, -1) -def load(core): - try: - pkl_file = open('links.pkl', 'rb') - obj = cPickle.load(pkl_file) - except: - obj = File_List() + def load(self): + try: + pkl_file = open('links.pkl', 'rb') + obj = cPickle.load(pkl_file) + except: + obj = {'version': LIST_VERSION} - if obj.version < LIST_VERSION: - obj = File_List() + if obj['version'] < LIST_VERSION: + obj = {'version': LIST_VERSION} - obj.set_core(core) + self.data = obj - return obj + diff --git a/module/thread_list.py b/module/thread_list.py index a7e92b77d..62131b20e 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -31,10 +31,11 @@ from download_thread import Download_Thread class Thread_List(object): def __init__(self, parent): self.parent = parent + self.list = parent.file_list self.threads = [] self.max_threads = int(self.parent.config['max_downloads']) - self.py_load_files = [] # files in queque - self.f_relation = [0, 0] + # self.py_load_files = [] # files in queque + # self.f_relation = [0, 0] self.lock = RLock() self.py_downloading = [] # files downloading self.occ_plugins = [] #occupied plugins @@ -50,36 +51,31 @@ class Thread_List(object): self.threads.append(thread) return True - def get_loaded_urls(self): - loaded_urls = [] - for file in self.py_load_files: - loaded_urls.append(file.url) - return loaded_urls - def remove_thread(self, thread): self.threads.remove(thread) + def select_thread(self): + """ select a thread + """ + while len(self.threads) < self.max_threads: + self.create_thread() + def get_job(self): """return job if suitable, otherwise send thread idle""" - if not self.parent.is_dltime(): + if not self.parent.is_dltime() or self.pause or self.reconnecting or not self.list.files: #conditions when threads dont download return None - if self.pause: - return None - - if self.reconnecting: - return None self.init_reconnect() self.lock.acquire() pyfile = None - for i in range(len(self.py_load_files)): - if not self.py_load_files[i].modul.__name__ in self.occ_plugins: - pyfile = self.py_load_files.pop(i) + for i in range(len(self.list.files)): + if not self.list.files[i].modul.__name__ in self.occ_plugins: + pyfile = self.list.files.pop(i) break if pyfile: @@ -103,25 +99,13 @@ class Thread_List(object): if pyfile.status.type == "finished": self.parent.logger.info('Download finished: ' + pyfile.url + ' @' + str(pyfile.status.get_speed()) + 'kb/s') - with open(self.parent.config['link_file'], 'r') as f: - data = f.read() - if pyfile.plugin.props['type'] == "container": - links = "" + + self.list.extend(pyfile.plugin.links) - for link in pyfile.plugin.links: - links += link + "\n" - - self.parent.extend_links(pyfile.plugin.links) - data = links + data # put the links into text file - - data = data.replace(pyfile.url + '\n', "") - - with open(self.parent.config['link_file'], 'w') as f: - f.write(data) if pyfile.status.type == "reconnected":#put it back in queque - self.py_load_files.insert(0, pyfile) + self.list.files.insert(0, pyfile) if pyfile.status.type == "failed": self.parent.logger.warning("Download failed: " + pyfile.url) @@ -131,18 +115,6 @@ class Thread_List(object): self.lock.release() return True - def select_thread(self): - """ select a thread - """ - if len(self.threads) < self.max_threads: - self.create_thread() - - def append_py_load_file(self, py_load_file): - py_load_file.id = len(self.py_load_files) - self.py_load_files.append(py_load_file) - self.f_relation[1] += 1 - self.select_thread() - def init_reconnect(self): """initialise a reonnect""" if not self.parent.config['use_reconnect']: |