(LogDB): initial commit

configure.ac

@@ -500,6 +500,7 @@ AC_CONFIG_FILES([Makefile
 				 extensions/tests/SMemoryTest/Makefile
 				 extensions/tests/MBSlaveTest/Makefile
 				 extensions/tests/MQPerfTest/Makefile
+				 extensions/LogDB/Makefile
 				 testsuite/Makefile
 				 wrappers/Makefile
 				 wrappers/python/lib/Makefile

extensions/LogDB/LogDB.cc

+/*
+ * Copyright (c) 2015 Pavel Vainerman.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation, version 2.1.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+// --------------------------------------------------------------------------
+/*! \file
+ *  \author Pavel Vainerman
+*/
+// --------------------------------------------------------------------------
+#include <sstream>
+#include <iomanip>
+
+#include "unisetstd.h"
+#include "LogDB.h"
+#include "Configuration.h"
+#include "Debug.h"
+#include "UniXML.h"
+#include "LogDBSugar.h"
+// --------------------------------------------------------------------------
+using namespace uniset;
+using namespace std;
+// --------------------------------------------------------------------------
+LogDB::LogDB( const string& name , const string& prefix ):
+	myname(name)
+{
+	dblog = make_shared<DebugStream>();
+
+	auto conf = uniset_conf();
+	auto xml = conf->getConfXML();
+
+	xmlNode* cnode = conf->findNode(xml->getFirstNode(), "LogDB", name);
+
+	if( !cnode )
+	{
+		ostringstream err;
+		err << name << "(init): Not found confnode <LogDB name='" << name << "'...>";
+		dbcrit << err.str() << endl;
+		throw uniset::SystemError(err.str());
+	}
+
+	UniXML::iterator it(cnode);
+
+	qbufSize = conf->getArgPInt("--" + prefix + "-buffer-size", it.getProp("bufferSize"), qbufSize);
+
+
+	UniXML::iterator sit(cnode);
+	if( !sit.goChildren() )
+	{
+		ostringstream err;
+		err << name << "(init): Not found confnode list of log servers for <LogDB name='" << name << "'...>";
+		dbcrit << err.str() << endl;
+		throw uniset::SystemError(err.str());
+	}
+
+	for( ;sit.getCurrent(); sit++ )
+	{
+		Log l;
+		l.name = sit.getProp("name");
+		l.ip = sit.getProp("ip");
+		l.port = sit.getIntProp("port");
+		l.cmd = sit.getProp("cmd");
+
+		if( l.name.empty()  )
+		{
+			ostringstream err;
+			err << name << "(init): Unknown name for logserver..";
+			dbcrit << err.str() << endl;
+			throw uniset::SystemError(err.str());
+		}
+
+		if( l.ip.empty()  )
+		{
+			ostringstream err;
+			err << name << "(init): Unknown 'ip' for '" << l.name << "'..";
+			dbcrit << err.str() << endl;
+			throw uniset::SystemError(err.str());
+		}
+
+		if( l.port == 0 )
+		{
+			ostringstream err;
+			err << name << "(init): Unknown 'port' for '" << l.name << "'..";
+			dbcrit << err.str() << endl;
+			throw uniset::SystemError(err.str());
+		}
+
+		if( l.cmd.empty() )
+		{
+			ostringstream err;
+			err << name << "(init): Unknown 'cmd' for '" << l.name << "'..";
+			dbcrit << err.str() << endl;
+			throw uniset::SystemError(err.str());
+		}
+
+		logservers.emplace_back(std::move(l));
+	}
+
+	if( logservers.empty() )
+	{
+		ostringstream err;
+		err << name << "(init): empty list of log servers for <LogDB name='" << name << "'...>";
+		dbcrit << err.str() << endl;
+		throw uniset::SystemError(err.str());
+	}
+
+
+
+
+	db = unisetstd::make_unique<SQLiteInterface>();
+
+#if 0
+	if( !db->connect(dbfile, false) )
+	{
+		//        ostringstream err;
+		dbcrit << myname
+			   << "(init): DB connection error: "
+			   << db->error() << endl;
+		//        throw Exception( string(myname+"(init): не смогли создать соединение с БД "+db->error()) );
+		askTimer(LogDB::ReconnectTimer, ReconnectTime);
+	}
+	else
+	{
+		dbinfo <<  myname << "(init): connect [OK]" << endl;
+		connect_ok = true;
+		askTimer(LogDB::ReconnectTimer, 0);
+		askTimer(LogDB::PingTimer, PingTime);
+		//        createTables(db);
+		initDB(db);
+		initDBTableMap(tblMap);
+		flushBuffer();
+	}
+#endif
+
+}
+//--------------------------------------------------------------------------------------------
+LogDB::~LogDB()
+{
+	if( db )
+		db->close();
+}
+//--------------------------------------------------------------------------------------------
+#if 0
+bool LogDB::writeToBase( const string& query )
+{
+	dbinfo <<  myname << "(writeToBase): " << query << endl;
+
+	//    cout << "LogDB: " << query << endl;
+	if( !db || !connect_ok )
+	{
+		uniset_rwmutex_wrlock l(mqbuf);
+		qbuf.push(query);
+
+		if( qbuf.size() > qbufSize )
+		{
+			std::string qlost;
+
+			if( lastRemove )
+				qlost = qbuf.back();
+			else
+				qlost = qbuf.front();
+
+			qbuf.pop();
+
+			dbcrit << myname << "(writeToBase): DB not connected! buffer(" << qbufSize
+				   << ") overflow! lost query: " << qlost << endl;
+		}
+
+		return false;
+	}
+
+	// На всякий скидываем очередь
+	flushBuffer();
+
+	// А теперь собственно запрос..
+	if( db->insert(query) )
+		return true;
+
+	return false;
+}
+#endif
+//--------------------------------------------------------------------------------------------
+void LogDB::flushBuffer()
+{
+	uniset_rwmutex_wrlock l(mqbuf);
+
+	// Сперва пробуем очистить всё что накопилось в очереди до этого...
+	while( !qbuf.empty() )
+	{
+		if( !db->insert(qbuf.front()) )
+		{
+			dbcrit << myname << "(flushBuffer): error: " << db->error() <<
+				   " lost query: " << qbuf.front() << endl;
+		}
+
+		qbuf.pop();
+	}
+}
+//--------------------------------------------------------------------------------------------
+std::shared_ptr<LogDB> LogDB::init_logdb( int argc, const char* const* argv, const std::string& prefix )
+{
+	auto conf = uniset_conf();
+
+	string name = conf->getArgParam("--" + prefix + "-name", "");
+
+	if( name.empty() )
+	{
+		cerr << "(LogDB): Unknown name. Use --" << prefix << "-name" << endl;
+		return nullptr;
+	}
+
+	return make_shared<LogDB>(name, prefix);
+}
+// -----------------------------------------------------------------------------
+void LogDB::help_print()
+{
+	cout << "Default: prefix='logdb'" << endl;
+	cout << "--prefix-name name        - Имя. Для поиска настроечной секции в configure.xml" << endl;
+	cout << "--prefix-buffer-size sz   - Размер буфера (до скидывания в БД)." << endl;
+}
+// -----------------------------------------------------------------------------
+void LogDB::run( bool async )
+{
+
+}
+// -----------------------------------------------------------------------------

extensions/LogDB/LogDB.h

+/*
+ * Copyright (c) 2015 Pavel Vainerman.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation, version 2.1.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+// --------------------------------------------------------------------------
+/*! \file
+ *  \author Pavel Vainerman
+*/
+// --------------------------------------------------------------------------
+#ifndef LogDB_H_
+#define LogDB_H_
+// --------------------------------------------------------------------------
+#include <queue>
+#include <memory>
+#include "UniSetTypes.h"
+#include "LogAgregator.h"
+#include "DebugStream.h"
+#include "SQLiteInterface.h"
+#include "LogReader.h"
+// -------------------------------------------------------------------------
+namespace uniset
+{
+	//------------------------------------------------------------------------------------------
+	/*!
+		  \page page_LogDB База логов
+
+		  - \ref sec_LogDB_Comm
+		  - \ref sec_LogDB_Conf
+		  - \ref sec_LogDB_REST
+
+
+		\section sec_LogDB_Comm Общее описание работы LogDB
+			LogDB это сервис, работа которого заключается
+		в подключении к указанным лог-серверам, получении от них логов
+		и сохранении их в БД (sqlite). Помимо этого LogDB выступает в качестве
+		REST сервиса, позволяющего получать логи за указанный период в виде json.
+
+		\section sec_LogDB_Conf Конфигурирвание LogDB
+
+		<LogDB name="LogDB" ...>
+			<logserver name="" ip=".." port=".." cmd=".."/>
+			<logserver name="" ip=".." port=".." cmd=".."/>
+			<logserver name="" ip=".." port=".." cmd=".."/>
+		</LogDB>
+
+
+		\section sec_LogDB_REST LogDB REST API
+	*/
+	class LogDB
+	{
+		public:
+			LogDB( const std::string& name, const std::string& prefix = "" );
+			virtual ~LogDB();
+
+			/*! глобальная функция для инициализации объекта */
+			static std::shared_ptr<LogDB> init_logdb( int argc, const char* const* argv, const std::string& prefix = "logdb" );
+
+			/*! глобальная функция для вывода help-а */
+			static void help_print();
+
+			inline std::shared_ptr<DebugStream> log()
+			{
+				return dblog;
+			}
+
+			void run( bool async );
+
+		protected:
+
+			std::string myname;
+			std::unique_ptr<SQLiteInterface> db;
+
+			bool activate = { false };
+
+			typedef std::queue<std::string> QueryBuffer;
+			QueryBuffer qbuf;
+			size_t qbufSize = { 200 }; // размер буфера сообщений.
+
+			void flushBuffer();
+			uniset::uniset_rwmutex mqbuf;
+
+			std::shared_ptr<DebugStream> dblog;
+
+			struct Log
+			{
+				std::string name;
+				std::string ip;
+				int port = { 0 };
+				std::string cmd;
+			};
+
+			std::vector<Log> logservers;
+
+		private:
+	};
+	// ----------------------------------------------------------------------------------
+} // end of namespace uniset
+//------------------------------------------------------------------------------------------
+#endif

extensions/LogDB/LogDBSugar.h

+#ifndef LogDBSugar_H_
+#define LogDBSugar_H_
+// "синтаксический сахар"..для логов
+#ifndef dbinfo
+#define dbinfo if( dblog->debugging(Debug::INFO) ) dblog->info()
+#endif
+#ifndef dbwarn
+#define dbwarn if( dblog->debugging(Debug::WARN) ) dblog->warn()
+#endif
+#ifndef dbcrit
+#define dbcrit if( dblog->debugging(Debug::CRIT) ) dblog->crit()
+#endif
+#ifndef dblog1
+#define dblog1 if( dblog->debugging(Debug::LEVEL1) ) dblog->level1()
+#endif
+#ifndef dblog2
+#define dblog2 if( dblog->debugging(Debug::LEVEL2) ) dblog->level2()
+#endif
+#ifndef dblog3
+#define dblog3 if( dblog->debugging(Debug::LEVEL3) ) dblog->level3()
+#endif
+#ifndef dblog4
+#define dblog4 if( dblog->debugging(Debug::LEVEL4) ) dblog->level4()
+#endif
+#ifndef dblog5
+#define dblog5 if( dblog->debugging(Debug::LEVEL5) ) dblog->level5()
+#endif
+#ifndef dblog6
+#define dblog6 if( dblog->debugging(Debug::LEVEL6) ) dblog->level6()
+#endif
+#ifndef dblog7
+#define dblog7 if( dblog->debugging(Debug::LEVEL7) ) dblog->level7()
+#endif
+#ifndef dblog8
+#define dblog8 if( dblog->debugging(Debug::LEVEL8) ) dblog->level8()
+#endif
+#ifndef dblog9
+#define dblog9 if( dblog->debugging(Debug::LEVEL9) ) dblog->level9()
+#endif
+#ifndef dblogany
+#define dblogany dblog->any()
+#endif
+#endif

extensions/LogDB/Makefile.am

+if DISABLE_SQLITE
+
+else
+
+bin_PROGRAMS = @PACKAGE@-logdb
+@PACKAGE@_logdb_LDADD = $(top_builddir)/extensions/DBServer-SQLite/libUniSet2-sqlite.la $(top_builddir)/lib/libUniSet2.la
+@PACKAGE@_logdb_CXXFLAGS = $(SQLITE3_CFLAGS) -I$(top_builddir)/extensions/DBServer-SQLite
+@PACKAGE@_logdb_SOURCES = LogDB.cc main.cc
+
+include $(top_builddir)/include.mk
+
+endif

extensions/LogDB/main.cc

+#include "Configuration.h"
+#include "LogDB.h"
+// --------------------------------------------------------------------------
+using namespace uniset;
+using namespace std;
+// --------------------------------------------------------------------------
+int main(int argc, char** argv)
+{
+	//	std::ios::sync_with_stdio(false);
+
+	try
+	{
+		if( argc > 1 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-h")) )
+		{
+			cout << "--confile filename       - configuration file. Default: configure.xml" << endl;
+			LogDB::help_print();
+			return 0;
+		}
+
+		uniset_init(argc, argv, "configure.xml");
+
+		auto db = LogDB::init_logdb(argc, argv);
+		db->run(false);
+	}
+	catch( const std::exception& ex )
+	{
+		cerr << "(LogDB::main): " << ex.what() << endl;
+	}
+	catch(...)
+	{
+		cerr << "(LogDB::main): catch ..." << endl;
+	}
+
+	return 0;
+}

extensions/LogDB/start_fg.sh

@@ -2,6 +2,4 @@
 
 ulimit -Sc 1000000
 
-uniset2-start.sh -f ./uniset2-sqlite-dbserver --confile test.xml --name DBServer1 \
---ulog-add-levels info,crit,warn,level9,system \
---dbserver-buffer-size 100
+uniset2-start.sh -f ./uniset2-logdb --confile test.xml --logdb-name LogDB

extensions/LogDB/stop.sh

+/usr/bin/uniset-stop.sh
\ No newline at end of file

extensions/LogDB/test.xml

+../../conf/test.xml
\ No newline at end of file

extensions/Makefile.am

@@ -6,7 +6,7 @@ if HAVE_EXTENTIONS
 SUBDIRS = lib include SharedMemory SharedMemory/tests IOControl IOControl/tests LogicProcessor LogicProcessor/tests \
 	ModbusMaster  ModbusSlave  SMViewer UniNetwork UNetUDP UNetUDP/tests \
 	DBServer-MySQL DBServer-SQLite DBServer-PostgreSQL MQTTPublisher \
-	RRDServer tests ModbusMaster/tests ModbusSlave/tests
+	RRDServer tests ModbusMaster/tests ModbusSlave/tests LogDB
 
 pkgconfigdir 	= $(libdir)/pkgconfig
 pkgconfig_DATA 	= libUniSet2Extensions.pc

uniset2.files

@@ -27,6 +27,11 @@ extensions/DBServer-PostgreSQL/Makefile.am
 extensions/DBServer-PostgreSQL/PostgreSQLInterface.cc
 extensions/DBServer-PostgreSQL/PostgreSQLInterface.h
 extensions/DBServer-PostgreSQL/test.cc
+extensions/LogDB/LogDB.cc
+extensions/LogDB/LogDB.h
+extensions/LogDB/LogDBSugar.h
+extensions/LogDB/main.cc
+extensions/LogDB/Makefile.am
 extensions/include/Calibration.h
 extensions/include/ComediInterface.h
 extensions/include/DigitalFilter.h