Version: 2.1 (using KDE KDE 3.5.5) Installed from: Fedora RPMs Compiler: gcc OS: Linux When torrenting a torrent that has many files (like 1000 or more) ktorrent eventually displays an error box saying too many open files and you have to restart the torrent for it to work again.
This particular torrent has 3724 files in it.
We are going to make sure this does not happen anymore in the 2.2 release.
SVN commit 640069 by guisson: Make sure to many files open problem is very unlikely to happen. BUG: 141454 M +6 -0 torrent/cache.cpp M +6 -0 torrent/cache.h M +15 -0 torrent/cachefile.cpp M +15 -8 torrent/multifilecache.cpp M +3 -0 torrent/peermanager.cpp M +2 -0 torrent/peermanager.h M +7 -0 util/functions.cpp M +3 -0 util/functions.h --- trunk/extragear/network/ktorrent/libktorrent/torrent/cache.cpp #640068:640069 @@ -20,6 +20,7 @@ #include "torrent.h" #include "chunk.h" #include "cache.h" +#include "peermanager.h" #include <util/functions.h> namespace bt @@ -46,4 +47,9 @@ { tmpdir = ndir; } + + bool Cache::mappedModeAllowed() + { + return MaxOpenFiles() - bt::PeerManager::getTotalConnections() < 100; + } } --- trunk/extragear/network/ktorrent/libktorrent/torrent/cache.h #640068:640069 @@ -124,6 +124,12 @@ * empty directories should also be deleted. */ virtual void deleteDataFiles() = 0; + + /** + * See if we are allowed to use mmap, when loading chunks. + * This will return false if we are close to system limits. + */ + static bool mappedModeAllowed(); }; } --- trunk/extragear/network/ktorrent/libktorrent/torrent/cachefile.cpp #640068:640069 @@ -293,12 +293,14 @@ void CacheFile::read(Uint8* buf,Uint32 size,Uint64 off) { QMutexLocker lock(&mutex); + bool close_again = false; // reopen the file if necessary if (fd == -1) { // Out() << "Reopening " << path << endl; openFile(); + close_again = true; } if (off >= file_size || off >= max_size) @@ -309,18 +311,28 @@ // jump to right position SeekFile(fd,off,SEEK_SET); if ((Uint32)::read(fd,buf,size) != size) + { + if (close_again) + closeTemporary(); + throw Error(i18n("Error reading from %1").arg(path)); + } + + if (close_again) + closeTemporary(); } void CacheFile::write(const Uint8* buf,Uint32 size,Uint64 off) { QMutexLocker lock(&mutex); + bool close_again = false; // reopen the file if necessary if (fd == -1) { // Out() << "Reopening " << path << endl; openFile(); + close_again = true; } if (off + size > max_size) @@ -338,6 +350,9 @@ // jump to right position SeekFile(fd,off,SEEK_SET); int ret = ::write(fd,buf,size); + if (close_again) + closeTemporary(); + if (ret == -1) throw Error(i18n("Error writing to %1 : %2").arg(path).arg(strerror(errno))); else if ((Uint32)ret != size) --- trunk/extragear/network/ktorrent/libktorrent/torrent/multifilecache.cpp #640068:640069 @@ -248,11 +248,18 @@ if (!fd) return; - Uint64 off = FileOffset(c,f,tor.getChunkSize()); - Uint8* buf = (Uint8*)fd->map(c,off,c->getSize(),CacheFile::READ); - if (buf) - c->setData(buf,Chunk::MMAPPED); - return; + if (Cache::mappedModeAllowed()) + { + Uint64 off = FileOffset(c,f,tor.getChunkSize()); + Uint8* buf = (Uint8*)fd->map(c,off,c->getSize(),CacheFile::READ); + if (buf) + { + c->setData(buf,Chunk::MMAPPED); + // only return when the mapping is OK + // if mmap fails we will just load it buffered + return; + } + } } Uint8* data = new Uint8[c->getSize()]; @@ -317,12 +324,12 @@ Uint64 off = FileOffset(c,tor.getFile(tflist.first()),tor.getChunkSize()); CacheFile* fd = files.find(tflist.first()); Uint8* buf = 0; - if (fd) + if (fd && Cache::mappedModeAllowed()) buf = (Uint8*)fd->map(c,off,c->getSize(),CacheFile::RW); + if (!buf) { - // if mmap fails use buffered mode - Out() << "Warning : mmap failed, falling back to buffered mode" << endl; + // if mmap fails or is not possible use buffered mode c->allocate(); c->setStatus(Chunk::BUFFERED); } --- trunk/extragear/network/ktorrent/libktorrent/torrent/peermanager.cpp #640068:640069 @@ -133,7 +133,10 @@ void PeerManager::setMaxTotalConnections(Uint32 max) { + Uint32 sys_max = bt::MaxOpenFiles() - 50; // leave about 50 free for regular files max_total_connections = max; + if (max == 0 || max_total_connections > sys_max) + max_total_connections = sys_max; } void PeerManager::addPotentialPeer(const PotentialPeer & pp) --- trunk/extragear/network/ktorrent/libktorrent/torrent/peermanager.h #640068:640069 @@ -126,6 +126,8 @@ static void setMaxTotalConnections(Uint32 max); static Uint32 getMaxTotalConnections() {return max_total_connections;} + static Uint32 getTotalConnections() {return total_connections;} + /// Is the peer manager started bool isStarted() const {return started;} --- trunk/extragear/network/ktorrent/libktorrent/util/functions.cpp #640068:640069 @@ -181,6 +181,13 @@ global_time_stamp = (Uint64)(tv.tv_sec * 1000 + tv.tv_usec * 0.001); return global_time_stamp; } + + Uint32 MaxOpenFiles() + { + struct rlimit lim; + getrlimit(RLIMIT_NOFILE,&lim); + return lim.rlim_cur; + } bool MaximizeLimits() { --- trunk/extragear/network/ktorrent/libktorrent/util/functions.h #640068:640069 @@ -64,6 +64,9 @@ * Maximize the file and memory limits using setrlimit. */ bool MaximizeLimits(); + + /// Get the maximum number of open files + Uint32 MaxOpenFiles(); } #endif
I tested a torrent with 22,000 files and I still get this error.
Which torrent ?
You have to log into pleasuredome.org.uk and try the "[FREE] MAME Extras v0.113" torrent. The torrent contains pictures of arcade games.
Though opening this torrent is rather slow, the to many files open error did not happen. Are you sure you tested this with the correct version ?
Oh I am using ktorrent 2.1.2, the 2.1.1 -> 2.1.2 occurred just a couple days ago, I mistook this as being the version that had the fix. Re-checking this bug I see you meant 2.2, not 2.1.2. I apologize for the confusion.
OK, then there is no reason to keep this bug open.