add upstream source (adch++ 2.12.1 from sourceforge)

src/plugins/Bloom/SConscript

@@ -0,0 +1,5 @@
+Import('dev source_path')
+
+ret = dev.build('src/')
+
+Return('ret')

src/plugins/Bloom/src/BloomManager.cpp

@@ -0,0 +1,220 @@
+/*
+ * Copyright (C) 2006-2016 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include "stdinc.h"
+#include "BloomManager.h"
+
+#include <adchpp/LogManager.h>
+#include <adchpp/Client.h>
+#include <adchpp/AdcCommand.h>
+#include <adchpp/Util.h>
+#include <adchpp/PluginManager.h>
+#include <adchpp/Core.h>
+
+using namespace std;
+using namespace std::placeholders;
+using namespace adchpp;
+
+const string BloomManager::className = "BloomManager";
+
+// TODO Make configurable
+const size_t h = 24;
+
+struct PendingItem {
+	PendingItem(size_t m_, size_t k_) : m(m_), k(k_) { buffer.reserve(m/8); }
+	ByteVector buffer;
+	size_t m;
+	size_t k;
+};
+
+BloomManager::BloomManager(Core &core) : searches(0), tthSearches(0), stopped(0), core(core) {
+	LOG(className, "Starting");
+}
+
+void BloomManager::init() {
+	auto &cm = core.getClientManager();
+	receiveConn = manage(cm.signalReceive().connect(std::bind(&BloomManager::onReceive, this, _1, _2, _3)));
+	sendConn = manage(cm.signalSend().connect(std::bind(&BloomManager::onSend, this, _1, _2, _3)));
+
+	auto &pm = core.getPluginManager();
+	bloomHandle = pm.registerPluginData(&PluginData::simpleDataDeleter<HashBloom>);
+	pendingHandle = pm.registerPluginData(&PluginData::simpleDataDeleter<PendingItem>);
+
+	statsConn = manage(pm.onCommand("stats", std::bind(&BloomManager::onStats, this, _1)));
+}
+
+bool BloomManager::hasBloom(Entity& c) const {
+	return c.getPluginData(bloomHandle);
+}
+
+bool BloomManager::hasTTH(Entity& c,const TTHValue& tth) const {
+	HashBloom* bloom = reinterpret_cast<HashBloom*>(c.getPluginData(bloomHandle));
+	return !bloom || bloom->match(tth);
+}
+
+int64_t BloomManager::getSearches() const {
+	return searches;
+}
+
+int64_t BloomManager::getTTHSearches() const {
+	return tthSearches;
+}
+
+int64_t BloomManager::getStoppedSearches() const {
+	return stopped;
+}
+
+BloomManager::~BloomManager() {
+	LOG(className, "Shutting down");
+}
+
+static const uint32_t FEATURE = AdcCommand::toFourCC("BLO0");
+
+void BloomManager::onReceive(Entity& e, AdcCommand& cmd, bool& ok) {
+	string tmp;
+
+	Client* cc = dynamic_cast<Client*>(&e);
+	if(!cc) {
+		return;
+	}
+
+	Client& c = *cc;
+	if(cmd.getCommand() == AdcCommand::CMD_INF && c.hasSupport(FEATURE)) {
+		if(cmd.getParam("SF", 0, tmp)) {
+			if(e.getPluginData(pendingHandle)) {
+				// Already getting a blom - we'll end up with an old bloom but there's no trivial
+				// way to avoid it...
+				// TODO Queue the blom get?
+				return;
+			}
+
+			size_t n = adchpp::Util::toInt(tmp);
+			if(n == 0) {
+				return;
+			}
+
+			e.clearPluginData(bloomHandle);
+
+			size_t k = HashBloom::get_k(n, h);
+			size_t m = HashBloom::get_m(n, k);
+
+			e.setPluginData(pendingHandle, new PendingItem(m, k));
+
+			AdcCommand get(AdcCommand::CMD_GET);
+			get.addParam("blom");
+			get.addParam("/");
+			get.addParam("0");
+			get.addParam(Util::toString(m/8));
+			get.addParam("BK", Util::toString(k));
+			get.addParam("BH", Util::toString(h));
+			c.send(get);
+		}
+	} else if(cmd.getCommand() == AdcCommand::CMD_SND) {
+		if(cmd.getParameters().size() < 4) {
+			return;
+		}
+
+		if(cmd.getParam(0) != "blom") {
+			return;
+		}
+
+		PendingItem* pending = reinterpret_cast<PendingItem*>(e.getPluginData(pendingHandle));
+		if(!pending) {
+			c.send(AdcCommand(AdcCommand::SEV_FATAL, AdcCommand::ERROR_BAD_STATE, "Unexpected bloom filter update"));
+			c.disconnect(Util::REASON_BAD_STATE);
+			ok = false;
+			return;
+		}
+
+		int64_t bytes = Util::toInt(cmd.getParam(3));
+
+		if(bytes != static_cast<int64_t>(pending->m / 8)) {
+			dcdebug("Disconnecting for invalid number of bytes: %d, %d\n", (int)bytes, (int)pending->m / 8);
+			c.send(AdcCommand(AdcCommand::SEV_FATAL, AdcCommand::ERROR_PROTOCOL_GENERIC, "Invalid number of bytes"));
+			c.disconnect(Util::REASON_PLUGIN);
+			ok = false;
+			e.clearPluginData(pendingHandle);
+			return;
+		}
+
+		c.setDataMode(bind(&BloomManager::onData, this, _1, _2, _3), bytes);
+		ok = false;
+	}
+}
+
+void BloomManager::onSend(Entity& c, const AdcCommand& cmd, bool& ok) {
+	if(!ok)
+		return;
+
+	if(cmd.getCommand() == AdcCommand::CMD_SCH) {
+		searches++;
+		string tmp;
+		if(cmd.getParam("TR", 0, tmp)) {
+			tthSearches++;
+			if(!hasTTH(c,TTHValue(tmp)) || !adchpp::Util::toInt(c.getField("SF"))) {
+				ok = false;
+				stopped++;
+			}
+		}
+	}
+}
+
+std::pair<size_t, size_t> BloomManager::getBytes() const {
+	std::pair<size_t, size_t> bytes;
+	auto &cm = core.getClientManager();
+	for(auto i = cm.getEntities().begin(), iend = cm.getEntities().end(); i != iend; ++i) {
+		auto bloom = reinterpret_cast<HashBloom*>(i->second->getPluginData(bloomHandle));
+		if(bloom) {
+			bytes.first++;
+			bytes.second += bloom->size();
+		}
+	}
+
+	return bytes;
+}
+
+void BloomManager::onData(Entity& c, const uint8_t* data, size_t len) {
+	PendingItem* pending = reinterpret_cast<PendingItem*>(c.getPluginData(pendingHandle));
+	if(!pending) {
+		// Shouldn't happen
+		return;
+	}
+
+	pending->buffer.insert(pending->buffer.end(), data, data + len);
+
+	if(pending->buffer.size() == pending->m / 8) {
+		HashBloom* bloom = new HashBloom();
+		c.setPluginData(bloomHandle, bloom);
+		bloom->reset(pending->buffer, pending->k, h);
+		c.clearPluginData(pendingHandle);
+		/* Mark the new filter as received */
+		signalBloomReady_(c);
+	}
+}
+
+void BloomManager::onStats(Entity& c) {
+	string stats = "\nBloom filter statistics:";
+	stats += "\nTotal outgoing searches: " + Util::toString(searches);
+	stats += "\nOutgoing TTH searches: " + Util::toString(tthSearches) + " (" + Util::toString(tthSearches * 100. / searches) + "% of total)";
+	stats += "\nStopped outgoing searches: " + Util::toString(stopped) + " (" + Util::toString(stopped * 100. / searches) + "% of total, " + Util::toString(stopped * 100. / tthSearches) + "% of TTH searches";
+	auto bytes = getBytes();
+	size_t clients = core.getClientManager().getEntities().size();
+	stats += "\nClient support: " + Util::toString(bytes.first) + "/" + Util::toString(clients) + " (" + Util::toString(bytes.first * 100. / clients) + "%)";
+	stats += "\nApproximate memory usage: " + Util::formatBytes(bytes.second) + ", " + Util::formatBytes(static_cast<double>(bytes.second) / clients) + "/client";
+	c.send(AdcCommand(AdcCommand::CMD_MSG).addParam(stats));
+}

src/plugins/Bloom/src/BloomManager.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2006-2016 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef BLOOM_MANAGER_H
+#define BLOOM_MANAGER_H
+
+#include <tuple>
+#include <adchpp/forward.h>
+#include <adchpp/Exception.h>
+#include <adchpp/ClientManager.h>
+#include <adchpp/Plugin.h>
+#include <adchpp/Signal.h>
+
+#include "HashBloom.h"
+
+STANDARD_EXCEPTION(BloomException);
+
+class ADCHPP_VISIBLE BloomManager : public Plugin {
+public:
+	BloomManager(Core &core);
+	virtual ~BloomManager();
+
+	virtual int getVersion() { return 2; }
+
+	void init();
+
+        /*Check if the entity has a bloom filter*/
+	PLUGIN_EXPORT bool hasBloom(Entity& c) const;
+        
+        /*Check if the entity may have the desired TTH according to the filter*/
+	PLUGIN_EXPORT bool hasTTH(Entity& c, const TTHValue& tth) const;
+
+	/*Get the number of searches sent (to clients)*/
+	PLUGIN_EXPORT int64_t getSearches() const;
+	/*Get the number of searches by TTH sent (to clients)*/
+	PLUGIN_EXPORT int64_t getTTHSearches() const;
+	/*Get the number of sent searches stopped*/
+	PLUGIN_EXPORT int64_t getStoppedSearches() const;
+        
+	static const std::string className;
+	
+	/*This signal is sent when a BloomFilter has been received*/
+	typedef SignalTraits<void (Entity&)> SignalBloomReady;
+	PLUGIN_EXPORT SignalBloomReady::Signal& signalBloomReady() { return signalBloomReady_; }
+
+private:
+	PluginDataHandle bloomHandle;
+	PluginDataHandle pendingHandle;
+
+	int64_t searches;
+	int64_t tthSearches;
+	int64_t stopped;
+
+	ClientManager::SignalReceive::ManagedConnection receiveConn;
+	ClientManager::SignalSend::ManagedConnection sendConn;
+	ClientManager::SignalReceive::ManagedConnection statsConn;
+
+	std::pair<size_t, size_t> getBytes() const;
+	void onReceive(Entity& c, AdcCommand& cmd, bool&);
+	void onSend(Entity& c, const AdcCommand& cmd, bool&);
+	void onData(Entity& c, const uint8_t* data, size_t len);
+	void onStats(Entity& c);
+
+	Core &core;
+	
+	SignalBloomReady::Signal signalBloomReady_;
+};
+
+#endif //ACCESSMANAGER_H

src/plugins/Bloom/src/BloomPlugin.cpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2006-2016 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include "stdinc.h"
+#include "BloomManager.h"
+
+#include <adchpp/PluginManager.h>
+
+#ifdef _WIN32
+
+BOOL APIENTRY DllMain(HANDLE /*hModule */, DWORD /* reason*/, LPVOID /*lpReserved*/) {
+    return TRUE;
+}
+
+#endif
+
+extern "C" {
+
+int PLUGIN_API pluginGetVersion() { return PLUGINVERSION; }
+
+int PLUGIN_API pluginLoad(PluginManager *pm) {
+	auto bm = make_shared<BloomManager>(pm->getCore());
+	bm->init();
+	pm->registerPlugin("BloomManager", bm);
+	return 0;
+}
+
+void PLUGIN_API pluginUnload() {
+
+}
+
+}
+

src/plugins/Bloom/src/HashBloom.cpp

@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2006-2016 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include "stdinc.h"
+
+#include "HashBloom.h"
+
+#include <cmath>
+
+size_t HashBloom::get_k(size_t n, size_t h) {
+	for(size_t k = TTHValue::BITS/h; k > 1; --k) {
+		uint64_t m = get_m(n, k);
+		if(m >> 24 == 0) {
+			return k;
+		}
+	}
+	return 1;
+}
+
+uint64_t HashBloom::get_m(size_t n, size_t k) {
+	uint64_t m = (static_cast<uint64_t>(ceil(static_cast<double>(n) * k / log(2.))));
+	// 64-bit boundary as per spec
+	return ((m + 63ULL) / 64ULL) * 64ULL;
+}
+
+void HashBloom::add(const TTHValue& tth) {
+	for(size_t i = 0; i < k; ++i) {
+		bloom[pos(tth, i)] = true;
+	}
+}
+
+bool HashBloom::match(const TTHValue& tth) const {
+	if(bloom.empty()) {
+		return false;
+	}
+	for(size_t i = 0; i < k; ++i) {
+		if(!bloom[pos(tth, i)]) {
+			return false;
+		}
+	}
+	return true;
+}
+
+void HashBloom::push_back(bool v) {
+	bloom.push_back(v);
+}
+
+void HashBloom::reset(ByteVector& v, size_t k_, size_t h_) {
+	k = k_;
+	h = h_;
+	
+	bloom.resize(v.size() * 8);
+	for(size_t i = 0; i < v.size(); ++i) {
+		for(size_t j = 0; j < 8; ++j) {
+			bloom[i*8 + j] = (((v[i] >> j) & 1) != 0);
+		}
+	}
+}
+
+size_t HashBloom::pos(const TTHValue& tth, size_t n) const {
+	if((n+1)*h > TTHValue::BITS) {
+		return 0;
+	}
+
+	uint64_t x = 0;
+	
+	size_t start = n * h;
+	
+	for(size_t i = 0; i < h; ++i) {
+		size_t bit = start + i;
+		size_t byte = bit / 8;
+		size_t pos = bit % 8;
+		
+		if(tth.data[byte] & (1 << pos)) {
+			x |= (1LL << i);
+		}
+	}
+	return x % bloom.size();
+}
+

src/plugins/Bloom/src/HashBloom.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2006-2016 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef HASHBLOOM_H_
+#define HASHBLOOM_H_
+
+#include "HashValue.h"
+
+/**
+ * According to http://www.eecs.harvard.edu/~michaelm/NEWWORK/postscripts/BloomFilterSurvey.pdf
+ * the optimal number of hashes k is (m/n)*ln(2), m = number of bits in the filter and n = number
+ * of items added. The largest k that we can get from a single TTH value depends on the number of 
+ * bits we need to address the bloom structure, which in turn depends on m, so the optimal size
+ * for our filter is m = n * k / ln(2) where n is the number of TTH values, or in our case, number of
+ * files in share since each file is identified by one TTH value. We try that for each even dividend 
+ * of the key size (2, 3, 4, 6, 8, 12) and if m fits within the bits we're able to address (2^(keysize/k)), 
+ * we can use that value when requesting the bloom filter.
+ */
+class HashBloom {
+public:
+	HashBloom() : k(0), h(0) { }
+	
+	/** Return a suitable value for k based on n */
+	static size_t get_k(size_t n, size_t h);
+	/** Optimal number of bits to allocate for n elements when using k hashes */
+	static uint64_t get_m(size_t n, size_t k);
+	
+	void add(const TTHValue& tth);
+	bool match(const TTHValue& tth) const;
+	void reset(ByteVector& v, size_t k, size_t h);
+	void push_back(bool v);
+	
+	size_t size() const { return bloom.size(); }
+private:	
+	
+	size_t pos(const TTHValue& tth, size_t n) const;
+	
+	std::vector<bool> bloom;
+	size_t k;
+	size_t h;
+};
+
+#endif /*HASHBLOOM_H_*/

src/plugins/Bloom/src/HashValue.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2001-2016 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef BLOOM_HASH_VALUE_H
+#define BLOOM_HASH_VALUE_H
+
+#include <adchpp/TigerHash.h>
+#include <adchpp/Encoder.h>
+
+template<class Hasher>
+struct HashValue {
+	static const size_t BITS = Hasher::BITS;
+	static const size_t BYTES = Hasher::BYTES;
+
+	HashValue() { }
+	explicit HashValue(uint8_t* aData) { memcpy(data, aData, BYTES); }
+	explicit HashValue(const std::string& base32) { Encoder::fromBase32(base32.c_str(), data, BYTES); }
+	HashValue(const HashValue& rhs) { memcpy(data, rhs.data, BYTES); }
+	HashValue& operator=(const HashValue& rhs) { memcpy(data, rhs.data, BYTES); return *this; }
+	bool operator!=(const HashValue& rhs) const { return !(*this == rhs); }
+	bool operator==(const HashValue& rhs) const { return memcmp(data, rhs.data, BYTES) == 0; }
+	bool operator<(const HashValue& rhs) const { return memcmp(data, rhs.data, BYTES) < 0; }
+
+	std::string toBase32() const { return Encoder::toBase32(data, BYTES); }
+	std::string& toBase32(std::string& tmp) const { return Encoder::toBase32(data, BYTES, tmp); }
+
+	uint8_t data[BYTES];
+};
+
+namespace std {
+template<typename T>
+struct hash<HashValue<T> > {
+	size_t operator()(const HashValue<T>& rhs) const { return *(size_t*)rhs.data; }
+};
+}
+
+typedef HashValue<TigerHash> TTHValue;
+
+#endif // !defined(HASH_VALUE_H)

src/plugins/Bloom/src/SConscript

@@ -0,0 +1,11 @@
+Import('dev source_path')
+
+env, target, sources = dev.prepare_build(source_path, 'Bloom', shared_precompiled_header = 'stdinc')
+env['SHLIBPREFIX'] = ''
+
+env.Append(CPPPATH = ['#'])
+env.Append(LIBS = ['adchpp'])
+
+ret = env.SharedLibrary(target, sources)
+
+Return('ret')

src/plugins/Bloom/src/stdinc.cpp

@@ -0,0 +1,19 @@
+/* 
+ * Copyright (C) 2006-2016 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include "stdinc.h"

src/plugins/Bloom/src/stdinc.h

@@ -0,0 +1,33 @@
+/* 
+ * Copyright (C) 2006-2016 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef BLOOM_STDINC_H
+#define BLOOM_STDINC_H
+
+#include <adchpp/adchpp.h>
+#include <adchpp/common.h>
+
+using namespace adchpp;
+
+#ifdef _WIN32
+#define PLUGIN_EXPORT __declspec(dllexport)
+#else
+#define PLUGIN_EXPORT __attribute__ ((visibility("default")))
+#endif
+
+#endif