diff options
Diffstat (limited to 'src/input')
-rw-r--r-- | src/input/BufferingInputStream.cxx | 12 | ||||
-rw-r--r-- | src/input/cache/Config.cxx | 35 | ||||
-rw-r--r-- | src/input/cache/Config.hxx | 33 | ||||
-rw-r--r-- | src/input/cache/Item.cxx | 62 | ||||
-rw-r--r-- | src/input/cache/Item.hxx | 80 | ||||
-rw-r--r-- | src/input/cache/Lease.hxx | 94 | ||||
-rw-r--r-- | src/input/cache/Manager.cxx | 163 | ||||
-rw-r--r-- | src/input/cache/Manager.hxx | 114 | ||||
-rw-r--r-- | src/input/cache/Stream.cxx | 96 | ||||
-rw-r--r-- | src/input/cache/Stream.hxx | 52 | ||||
-rw-r--r-- | src/input/meson.build | 4 |
11 files changed, 745 insertions, 0 deletions
diff --git a/src/input/BufferingInputStream.cxx b/src/input/BufferingInputStream.cxx index 8888f597c..9bdb91b49 100644 --- a/src/input/BufferingInputStream.cxx +++ b/src/input/BufferingInputStream.cxx @@ -152,6 +152,18 @@ BufferingInputStream::RunThreadLocked(std::unique_lock<Mutex> &lock) continue; } + /* enforce an upper limit for each + InputStream::Read() call; this is necessary + for plugins which are unable to do partial + reads, e.g. when reading local files, the + read() system call will not return until + all requested bytes have been read from the + hard disk, instead of returning when "some" + data has been read */ + constexpr size_t MAX_READ = 64 * 1024; + if (w.size > MAX_READ) + w.size = MAX_READ; + size_t nbytes = input->Read(lock, w.data, w.size); buffer.Commit(read_offset, read_offset + nbytes); diff --git a/src/input/cache/Config.cxx b/src/input/cache/Config.cxx new file mode 100644 index 000000000..efb472497 --- /dev/null +++ b/src/input/cache/Config.cxx @@ -0,0 +1,35 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "Config.hxx" +#include "config/Block.hxx" +#include "config/Parser.hxx" + +static constexpr size_t KILOBYTE = 1024; +static constexpr size_t MEGABYTE = 1024 * KILOBYTE; + +InputCacheConfig::InputCacheConfig(const ConfigBlock &block) +{ + size = 256 * MEGABYTE; + const auto *size_param = block.GetBlockParam("size"); + if (size_param != nullptr) + size = size_param->With([](const char *s){ + return ParseSize(s); + }); +} diff --git a/src/input/cache/Config.hxx b/src/input/cache/Config.hxx new file mode 100644 index 000000000..3ea1c551c --- /dev/null +++ b/src/input/cache/Config.hxx @@ -0,0 +1,33 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MPD_INPUT_CACHE_CONFIG_HXX +#define MPD_INPUT_CACHE_CONFIG_HXX + +#include <stddef.h> + +struct ConfigBlock; + +struct InputCacheConfig { + size_t size; + + explicit InputCacheConfig(const ConfigBlock &block); +}; + +#endif diff --git a/src/input/cache/Item.cxx b/src/input/cache/Item.cxx new file mode 100644 index 000000000..5decaaced --- /dev/null +++ b/src/input/cache/Item.cxx @@ -0,0 +1,62 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "Item.hxx" +#include "Lease.hxx" + +#include <assert.h> + +InputCacheItem::InputCacheItem(InputStreamPtr _input) noexcept + :BufferingInputStream(std::move(_input)), + uri(GetInput().GetURI()) +{ +} + +InputCacheItem::~InputCacheItem() noexcept +{ + assert(leases.empty()); +} + +void +InputCacheItem::AddLease(InputCacheLease &lease) noexcept +{ + const std::lock_guard<Mutex> lock(mutex); + leases.push_back(lease); +} + +void +InputCacheItem::RemoveLease(InputCacheLease &lease) noexcept +{ + const std::lock_guard<Mutex> lock(mutex); + auto i = leases.iterator_to(lease); + if (i == next_lease) + ++next_lease; + leases.erase(i); + + // TODO: ensure that OnBufferAvailable() isn't currently running +} + +void +InputCacheItem::OnBufferAvailable() noexcept +{ + for (auto i = leases.begin(); i != leases.end(); i = next_lease) { + next_lease = std::next(i); + i->OnInputCacheAvailable(); + } +} diff --git a/src/input/cache/Item.hxx b/src/input/cache/Item.hxx new file mode 100644 index 000000000..53b9668a1 --- /dev/null +++ b/src/input/cache/Item.hxx @@ -0,0 +1,80 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MPD_INPUT_CACHE_ITEM_HXX +#define MPD_INPUT_CACHE_ITEM_HXX + +#include "input/BufferingInputStream.hxx" +#include "thread/Mutex.hxx" +#include "util/SparseBuffer.hxx" + +#include <boost/intrusive/list.hpp> +#include <boost/intrusive/set_hook.hpp> + +#include <memory> +#include <string> + +class InputCacheLease; + +/** + * An item in the #InputCacheManager. It caches the contents of a + * file, and reading and managing it through the base class + * #BufferingInputStream. + * + * Use the class #CacheInputStream to read from it. + */ +class InputCacheItem final + : public BufferingInputStream, + public boost::intrusive::list_base_hook<boost::intrusive::link_mode<boost::intrusive::auto_unlink>>, + public boost::intrusive::set_base_hook<boost::intrusive::link_mode<boost::intrusive::normal_link>> +{ + const std::string uri; + + using LeaseList = + boost::intrusive::list<InputCacheLease, + boost::intrusive::base_hook<boost::intrusive::list_base_hook<boost::intrusive::link_mode<boost::intrusive::normal_link>>>, + boost::intrusive::constant_time_size<false>>; + + LeaseList leases; + LeaseList::iterator next_lease = leases.end(); + +public: + explicit InputCacheItem(InputStreamPtr _input) noexcept; + ~InputCacheItem() noexcept; + + const char *GetUri() const noexcept { + return uri.c_str(); + } + + using BufferingInputStream::size; + + bool IsInUse() const noexcept { + const std::lock_guard<Mutex> lock(mutex); + return !leases.empty(); + } + + void AddLease(InputCacheLease &lease) noexcept; + void RemoveLease(InputCacheLease &lease) noexcept; + +private: + /* virtual methods from class BufferingInputStream */ + void OnBufferAvailable() noexcept override; +}; + +#endif diff --git a/src/input/cache/Lease.hxx b/src/input/cache/Lease.hxx new file mode 100644 index 000000000..be0380e84 --- /dev/null +++ b/src/input/cache/Lease.hxx @@ -0,0 +1,94 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MPD_INPUT_CACHE_LEASE_HXX +#define MPD_INPUT_CACHE_LEASE_HXX + +#include "Item.hxx" + +#include <boost/intrusive/list_hook.hpp> + +#include <utility> + +/** + * A lease for an #InputCacheItem. + */ +class InputCacheLease + : public boost::intrusive::list_base_hook<boost::intrusive::link_mode<boost::intrusive::normal_link>> +{ + InputCacheItem *item = nullptr; + +public: + InputCacheLease() = default; + + explicit InputCacheLease(InputCacheItem &_item) noexcept + :item(&_item) + { + item->AddLease(*this); + } + + InputCacheLease(InputCacheLease &&src) noexcept + :item(std::exchange(src.item, nullptr)) + { + if (item != nullptr) { + item->RemoveLease(src); + item->AddLease(*this); + } + } + + ~InputCacheLease() noexcept { + if (item != nullptr) + item->RemoveLease(*this); + } + + InputCacheLease &operator=(InputCacheLease &&src) noexcept { + using std::swap; + swap(item, src.item); + + if (item != nullptr) { + item->RemoveLease(src); + item->AddLease(*this); + } + + return *this; + } + + operator bool() const noexcept { + return item != nullptr; + } + + auto &operator*() const noexcept { + return *item; + } + + auto *operator->() const noexcept { + return item; + } + + auto &GetCacheItem() const noexcept { + return *item; + } + + /** + * Caller locks #InputCacheItem::mutex. + */ + virtual void OnInputCacheAvailable() noexcept {} +}; + +#endif diff --git a/src/input/cache/Manager.cxx b/src/input/cache/Manager.cxx new file mode 100644 index 000000000..3ff51d88e --- /dev/null +++ b/src/input/cache/Manager.cxx @@ -0,0 +1,163 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "Manager.hxx" +#include "Config.hxx" +#include "Item.hxx" +#include "Lease.hxx" +#include "input/InputStream.hxx" +#include "fs/Traits.hxx" +#include "util/DeleteDisposer.hxx" + +#include <string.h> + +inline bool +InputCacheManager::ItemCompare::operator()(const InputCacheItem &a, + const char *b) const noexcept +{ + return strcmp(a.GetUri(), b) < 0; +} + +inline bool +InputCacheManager::ItemCompare::operator()(const char *a, + const InputCacheItem &b) const noexcept +{ + return strcmp(a, b.GetUri()) < 0; +} + +inline bool +InputCacheManager::ItemCompare::operator()(const InputCacheItem &a, + const InputCacheItem &b) const noexcept +{ + return strcmp(a.GetUri(), b.GetUri()) < 0; +} + +InputCacheManager::InputCacheManager(const InputCacheConfig &config) noexcept + :max_total_size(config.size) +{ +} + +InputCacheManager::~InputCacheManager() noexcept +{ + items_by_time.clear_and_dispose(DeleteDisposer()); +} + +bool +InputCacheManager::IsEligible(const InputStream &input) noexcept +{ + assert(input.IsReady()); + + return input.IsSeekable() && input.KnownSize() && + input.GetSize() > 0 && + input.GetSize() <= max_total_size / 2; +} + +bool +InputCacheManager::Contains(const char *uri) noexcept +{ + return Get(uri, false); +} + +InputCacheLease +InputCacheManager::Get(const char *uri, bool create) +{ + // TODO: allow caching remote files + if (!PathTraitsUTF8::IsAbsolute(uri)) + return {}; + + UriMap::insert_commit_data hint; + auto result = items_by_uri.insert_check(uri, items_by_uri.key_comp(), + hint); + if (!result.second) { + auto &item = *result.first; + + /* refresh */ + items_by_time.erase(items_by_time.iterator_to(item)); + items_by_time.push_back(item); + + // TODO revalidate the cache item using the file's mtime? + // TODO if cache item contains error, retry now? + + return InputCacheLease(item); + } + + if (!create) + return {}; + + // TODO: wait for "ready" without blocking here + auto is = InputStream::OpenReady(uri, mutex); + + if (!IsEligible(*is)) + return {}; + + const size_t size = is->GetSize(); + total_size += size; + + while (total_size > max_total_size && EvictOldestUnused()) {} + + auto *item = new InputCacheItem(std::move(is)); + items_by_uri.insert_commit(*item, hint); + items_by_time.push_back(*item); + + return InputCacheLease(*item); +} + +void +InputCacheManager::Prefetch(const char *uri) +{ + Get(uri, true); +} + +void +InputCacheManager::Remove(InputCacheItem &item) noexcept +{ + assert(total_size >= item.size()); + total_size -= item.size(); + + items_by_time.erase(items_by_time.iterator_to(item)); + items_by_uri.erase(items_by_uri.iterator_to(item)); +} + +void +InputCacheManager::Delete(InputCacheItem *item) noexcept +{ + Remove(*item); + delete item; +} + +InputCacheItem * +InputCacheManager::FindOldestUnused() noexcept +{ + for (auto &i : items_by_time) + if (!i.IsInUse()) + return &i; + + return nullptr; +} + +bool +InputCacheManager::EvictOldestUnused() noexcept +{ + auto *item = FindOldestUnused(); + if (item == nullptr) + return false; + + Delete(item); + return true; +} diff --git a/src/input/cache/Manager.hxx b/src/input/cache/Manager.hxx new file mode 100644 index 000000000..6fba1fbb3 --- /dev/null +++ b/src/input/cache/Manager.hxx @@ -0,0 +1,114 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MPD_INPUT_CACHE_MANAGER_HXX +#define MPD_INPUT_CACHE_MANAGER_HXX + +#include "input/Offset.hxx" +#include "thread/Mutex.hxx" +#include "util/Compiler.h" + +#include <boost/intrusive/set.hpp> +#include <boost/intrusive/list.hpp> + +class InputStream; +class InputCacheItem; +class InputCacheLease; +struct InputCacheConfig; + +/** + * A class which caches files in RAM. It is supposed to prefetch + * files before they are played. + */ +class InputCacheManager { + const size_t max_total_size; + + mutable Mutex mutex; + + size_t total_size = 0; + + struct ItemCompare { + gcc_pure + bool operator()(const InputCacheItem &a, + const char *b) const noexcept; + + gcc_pure + bool operator()(const char *a, + const InputCacheItem &b) const noexcept; + + gcc_pure + bool operator()(const InputCacheItem &a, + const InputCacheItem &b) const noexcept; + }; + + boost::intrusive::list<InputCacheItem, + boost::intrusive::base_hook<boost::intrusive::list_base_hook<boost::intrusive::link_mode<boost::intrusive::auto_unlink>>>, + boost::intrusive::constant_time_size<false>> items_by_time; + + using UriMap = + boost::intrusive::set<InputCacheItem, + boost::intrusive::base_hook<boost::intrusive::set_base_hook<boost::intrusive::link_mode<boost::intrusive::normal_link>>>, + boost::intrusive::compare<ItemCompare>, + boost::intrusive::constant_time_size<false>>; + + UriMap items_by_uri; + +public: + explicit InputCacheManager(const InputCacheConfig &config) noexcept; + ~InputCacheManager() noexcept; + + gcc_pure + bool Contains(const char *uri) noexcept; + + /** + * Throws if opening the #InputStream fails. + * + * @param create if true, then the cache item will be created + * if it did not exist + * @return a lease of the new item or nullptr if the file is + * not eligible for caching + */ + InputCacheLease Get(const char *uri, bool create); + + /** + * Shortcut for "Get(uri,true)", discarding the returned + * lease. + */ + void Prefetch(const char *uri); + +private: + /** + * Check whether the given #InputStream can be stored in this + * cache. + */ + bool IsEligible(const InputStream &input) noexcept; + + void Remove(InputCacheItem &item) noexcept; + void Delete(InputCacheItem *item) noexcept; + + InputCacheItem *FindOldestUnused() noexcept; + + /** + * @return true if one item has been evicted, false if no + * unused item was found + */ + bool EvictOldestUnused() noexcept; +}; + +#endif diff --git a/src/input/cache/Stream.cxx b/src/input/cache/Stream.cxx new file mode 100644 index 000000000..5ce56eaef --- /dev/null +++ b/src/input/cache/Stream.cxx @@ -0,0 +1,96 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "Stream.hxx" + +CacheInputStream::CacheInputStream(InputCacheLease _lease, + Mutex &_mutex) noexcept + :InputStream(_lease->GetUri(), _mutex), + InputCacheLease(std::move(_lease)) +{ + auto &i = GetCacheItem(); + size = i.size(); + seekable = true; + SetReady(); +} + +void +CacheInputStream::Check() +{ + const ScopeUnlock unlock(mutex); + + auto &i = GetCacheItem(); + const std::lock_guard<Mutex> protect(i.mutex); + + i.Check(); +} + +void +CacheInputStream::Seek(std::unique_lock<Mutex> &, offset_type new_offset) +{ + offset = new_offset; +} + +bool +CacheInputStream::IsEOF() const noexcept +{ + return offset == size; +} + +bool +CacheInputStream::IsAvailable() const noexcept +{ + const auto _offset = offset; + const ScopeUnlock unlock(mutex); + + auto &i = GetCacheItem(); + const std::lock_guard<Mutex> protect(i.mutex); + + return i.IsAvailable(_offset); +} + +size_t +CacheInputStream::Read(std::unique_lock<Mutex> &lock, + void *ptr, size_t read_size) +{ + const auto _offset = offset; + auto &i = GetCacheItem(); + + size_t nbytes; + + { + const ScopeUnlock unlock(mutex); + const std::lock_guard<Mutex> protect(i.mutex); + + nbytes = i.Read(lock, _offset, ptr, read_size); + } + + offset += nbytes; + return nbytes; +} + +void +CacheInputStream::OnInputCacheAvailable() noexcept +{ + auto &i = GetCacheItem(); + const ScopeUnlock unlock(i.mutex); + + const std::lock_guard<Mutex> protect(mutex); + InvokeOnAvailable(); +} diff --git a/src/input/cache/Stream.hxx b/src/input/cache/Stream.hxx new file mode 100644 index 000000000..ae039359c --- /dev/null +++ b/src/input/cache/Stream.hxx @@ -0,0 +1,52 @@ +/* + * Copyright 2003-2019 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MPD_CACHE_INPUT_STREAM_HXX +#define MPD_CACHE_INPUT_STREAM_HXX + +#include "Lease.hxx" +#include "input/InputStream.hxx" + +/** + * An #InputStream implementation which reads data from an + * #InputCacheItem. + */ +class CacheInputStream final : public InputStream, InputCacheLease { +public: + CacheInputStream(InputCacheLease _lease, Mutex &_mutex) noexcept; + + /* virtual methods from class InputStream */ + void Check() override; + /* we don't need to implement Update() because all attributes + have been copied already in our constructor */ + //void Update() noexcept; + void Seek(std::unique_lock<Mutex> &lock, offset_type offset) override; + bool IsEOF() const noexcept override; + /* we don't support tags */ + // std::unique_ptr<Tag> ReadTag() override; + bool IsAvailable() const noexcept override; + size_t Read(std::unique_lock<Mutex> &lock, + void *ptr, size_t size) override; + +private: + /* virtual methods from class InputCacheLease */ + void OnInputCacheAvailable() noexcept override; +}; + +#endif diff --git a/src/input/meson.build b/src/input/meson.build index 3502379d8..5f7533751 100644 --- a/src/input/meson.build +++ b/src/input/meson.build @@ -35,6 +35,10 @@ input_glue = static_library( 'BufferingInputStream.cxx', 'BufferedInputStream.cxx', 'MaybeBufferedInputStream.cxx', + 'cache/Config.cxx', + 'cache/Manager.cxx', + 'cache/Item.cxx', + 'cache/Stream.cxx', include_directories: inc, ) |