Skip to content

U
uniset2
Commit dbfe112f authored by Pavel Vainerman's avatar Pavel Vainerman
Browse files
Options

(LogDB): добавил утилиту конвертирования логов в БД.

parent 1483b4a9
Hide whitespace changes
Inline Side-by-side
Showing with 236 additions and 9 deletions
extensions/LogDB/LogDB.cc
View file @ dbfe112f
......@@ -63,7 +63,8 @@ LogDB::LogDB( const string& name , const string& prefix ):
std::string s_overflow = conf->getArg2Param("--" + prefix + "overflow-factor", it.getProp("overflowFactor"), "1.3");
float ovf = atof(s_overflow.c_str());
numOverflow = lroundf( (float)maxdbRecords*ovf );
numOverflow = lroundf( (float)maxdbRecords * ovf );
if( numOverflow == 0 )
numOverflow = maxdbRecords;
......@@ -213,7 +214,7 @@ void LogDB::flushBuffer()
db->query("COMMIT;");
if( !db->error().empty() )
if( !db->lastQueryOK() )
{
dbcrit << myname << "(flushBuffer): error: " << db->error() << endl;
}
......@@ -238,18 +239,20 @@ void LogDB::rotateDB()
size_t firstOldID = getFirstOfOldRecord(numOverflow);
DBResult ret = db->query("DELETE FROM logs WHERE id <= " + std::to_string(firstOldID) + ";");
if( !db->lastQueryOK() )
{
dbwarn << myname << "(rotateDB): delete error: " << db->error() << endl;
}
ret = db->query("VACUUM;");
if( !db->lastQueryOK() )
{
dbwarn << myname << "(rotateDB): vacuum error: " << db->error() << endl;
}
// dblog3 << myname << "(rotateDB): after rotate: " << getCountOfRecords() << " records" << endl;
// dblog3 << myname << "(rotateDB): after rotate: " << getCountOfRecords() << " records" << endl;
}
//--------------------------------------------------------------------------------------------
void LogDB::addLog( LogDB::Log* log, const string& txt )
......@@ -272,6 +275,7 @@ size_t LogDB::getCountOfRecords( const std::string& logname )
ostringstream q;
q << "SELECT count(*) FROM logs";
if( !logname.empty() )
q << " WHERE name='" << logname << "'";
......@@ -280,7 +284,7 @@ size_t LogDB::getCountOfRecords( const std::string& logname )
if( !ret )
return 0;
return (size_t) DBResult::as_int(ret.begin(),0);
return (size_t) DBResult::as_int(ret.begin(), 0);
}
//--------------------------------------------------------------------------------------------
size_t LogDB::getFirstOfOldRecord( size_t maxnum )
......@@ -293,7 +297,7 @@ size_t LogDB::getFirstOfOldRecord( size_t maxnum )
if( !ret )
return 0;
return (size_t) DBResult::as_int(ret.begin(),0);
return (size_t) DBResult::as_int(ret.begin(), 0);
}
//--------------------------------------------------------------------------------------------
std::shared_ptr<LogDB> LogDB::init_logdb( int argc, const char* const* argv, const std::string& prefix )
......@@ -883,7 +887,7 @@ Poco::JSON::Object::Ptr LogDB::httpGetCount( const Poco::URI::QueryParameters& p
size_t count = getCountOfRecords(logname);
jdata->set("name", logname);
jdata->set("count",count);
jdata->set("count", count);
return jdata;
}
// -----------------------------------------------------------------------------
......
extensions/LogDB/LogDB.h
View file @ dbfe112f
......@@ -128,6 +128,8 @@ namespace uniset
\todo WebSocket: доделать настройку всевозможных timeout-ов
\todo WebSocket: доделать проверку соединения
\todo WebSocket: сделать ограничение на максимальное количество соединений
\todo utils: написать конвертор обычных uniset-логов в БД.
\todo db: возможно всё-таки стоит парсить логи на предмет loglevel, и тогда уж и дату с временем вынимать
*/
class LogDB:
public EventLoopServer
......@@ -169,7 +171,7 @@ namespace uniset
void onActivate( ev::async& watcher, int revents ) ;
void addLog( Log* log, const std::string& txt );
size_t getCountOfRecords( const std::string& logname="" );
size_t getCountOfRecords( const std::string& logname = "" );
size_t getFirstOfOldRecord( size_t maxnum );
#ifndef DISABLE_REST_API
......@@ -200,7 +202,7 @@ namespace uniset
void flushBuffer();
void rotateDB();
size_t maxdbRecords = { 200*1000 };
size_t maxdbRecords = { 200 * 1000 };
size_t numOverflow = { 0 }; // вычисляется из параметра "overflow factor"(float)
ev::async wsactivate; // активация LogWebSocket-ов
......
extensions/LogDB/Makefile.am
View file @ dbfe112f
......@@ -4,11 +4,15 @@ else
bin_SCRIPTS = @PACKAGE@-logdb-create
bin_PROGRAMS = @PACKAGE@-logdb
bin_PROGRAMS = @PACKAGE@-logdb @PACKAGE@-logdb-conv
@PACKAGE@_logdb_LDADD = $(top_builddir)/extensions/DBServer-SQLite/libUniSet2-sqlite.la $(top_builddir)/lib/libUniSet2.la
@PACKAGE@_logdb_CXXFLAGS = $(SQLITE3_CFLAGS) -I$(top_builddir)/extensions/DBServer-SQLite
@PACKAGE@_logdb_SOURCES = LogDB.cc main.cc
@PACKAGE@_logdb_conv_LDADD = $(top_builddir)/extensions/DBServer-SQLite/libUniSet2-sqlite.la $(top_builddir)/lib/libUniSet2.la
@PACKAGE@_logdb_conv_CXXFLAGS = $(SQLITE3_CFLAGS) -I$(top_builddir)/extensions/DBServer-SQLite
@PACKAGE@_logdb_conv_SOURCES = logdb-conv.cc
include $(top_builddir)/include.mk
endif
extensions/LogDB/logdb-conv.cc 0 → 100644
View file @ dbfe112f
#include <string>
#include <vector>
#include <fstream>
#include <regex>
#include "SQLiteInterface.h"
// --------------------------------------------------------------------------
using namespace uniset;
using namespace std;
// --------------------------------------------------------------------------
typedef std::pair<std::string, std::string> LogInfo;
const size_t maxRead = 5000; // сколько читать прежде чем записать в БД
std::vector<std::string> qbuf;
// --------------------------------------------------------------------------
void saveToDB( SQLiteInterface* db );
void parseLine( SQLiteInterface* db, const LogInfo& loginfo, const std::string& line );
void parseFile( SQLiteInterface* db, const LogInfo& loginfo );
LogInfo makeLogInfo( const std::string& name );
// --------------------------------------------------------------------------
int main(int argc, char** argv)
{
try
{
if( argc < 3 )
{
cout << "uniset2-logdb-conv dbfile [logname1:]logfile1.log [logname2:]logfile2.log..." << endl;
cout << " logfile - log file" << endl;
cout << " [lognameX] - log name for db. Default: name of logfile" << endl;
cout << " : - separator for logname" << endl;
cout << endl;
cout << "log format: " << endl;
cout << "28/07/2017 10:55:19( info): xxx text text text" << endl;
cout << "28/07/2017 10:55:19( info): xxx text text text" << endl;
cout << "28/07/2017 10:55:19( info): xxx text text text" << endl;
cout << endl;
return 0;
}
std::string dbfile = argv[1];
std::vector<LogInfo> files;
for( int i = 2; i < argc; i++ )
files.emplace_back(makeLogInfo(argv[i]));
SQLiteInterface db;
if( !db.connect(dbfile, false) )
{
cerr << "(logdb-conv): DB connection error: " << db.error() << endl;
return 1;
}
qbuf.reserve(maxRead);
for( const auto& f : files )
parseFile(&db, f);
db.close();
return 0;
}
catch( const std::exception& ex )
{
cerr << "(logdb-conv): " << ex.what() << endl;
}
catch(...)
{
cerr << "(logdb-conv): catch ..." << endl;
}
return 1;
}
// --------------------------------------------------------------------------
LogInfo makeLogInfo( const std::string& name )
{
LogInfo linf;
auto pos = name.find(':');
if( pos != string::npos )
{
linf.first = name.substr(0, pos);
linf.second = name.substr(pos + 1, name.size());
}
else
{
linf.second = name;
// вырезаем расширение файла
auto epos = name.rfind('.');
if( epos == string::npos )
linf.first = name;
else
linf.first = name.substr(0, epos);
}
return linf;
}
// --------------------------------------------------------------------------
void parseLine( SQLiteInterface* db, const LogInfo& loginfo, const std::string& line )
{
// 28/07/2017 10:55:19( info): QG3(AutomatTransientMode): ждем отключения автомата
static const std::regex re("[::space::]{0,}"
"(\\d{2})[/-]{1}(\\d{2})[/-]{1}(\\d{4})"
" "
"(\\d{2}:\\d{2}:\\d{2})"
// "(.*)$"
);
std::smatch match;
if( !std::regex_search(line, match, re) )
{
cerr << "error: parse line: " << line << endl;
return;
}
if( match.size() < 5 )
{
cerr << "error: parse line: " << line << endl;
return;
}
ostringstream q;
q << "INSERT INTO logs(tms,usec,name,text) VALUES("
<< "CAST( strftime('%s','" << match[3] << "-" << match[2] << "-" << match[1] << " " << match[4] << "') AS INT)"
<< ",0,'" // usec
<< loginfo.first << "','" // name
<< line << "');"; // text
qbuf.push_back(q.str());
if( qbuf.size() >= maxRead )
saveToDB(db);
}
// --------------------------------------------------------------------------
void parseFile( SQLiteInterface* db, const LogInfo& loginfo )
{
ifstream f(loginfo.second);
if( !f.is_open() )
{
cerr << "(logdb-conv): can`t open "
<< loginfo.second << ". error: " << strerror(errno)
<< endl;
return;
}
std::string line;
while( std::getline(f, line) )
parseLine(db, loginfo, line);
f.close();
saveToDB(db);
}
// --------------------------------------------------------------------------
void saveToDB( SQLiteInterface* db )
{
db->query("BEGIN;");
for( const auto& q : qbuf )
{
if( !db->insert(q) )
{
cerr << "(saveToDB): error: " << db->error()
<< " lost log: " << q << endl;
}
}
db->query("COMMIT;");
if( !db->lastQueryOK() )
cerr << "(saveToDB): error: " << db->error() << endl;
qbuf.clear();
qbuf.reserve(maxRead);
}
// --------------------------------------------------------------------------
extensions/LogDB/test.log 0 → 100644
View file @ dbfe112f
08/09/2017 14:54:29( info): Object1(make_child_object): create Object8...
08/09/2017 14:54:29( info): Object1(make_child_object): create Object7...
08/09/2017 14:54:29( crit): Object2(setdata): UI(setValue): resolve failed (6518:60000)localhost
08/09/2017 14:54:29( crit): Object2(setOut): UI(setValue): resolve failed (6518:60000)localhost
08/09/2017 14:54:29( crit): Object2(setdata): UI(setValue): resolve failed (6519:60000)localhost
08/09/2017 14:54:29( crit): Object2(setOut): UI(setValue): resolve failed (6519:60000)localhost
08/09/2017 14:54:29( crit): Object2(setdata): UI(setValue): resolve failed (6521:60000)localhost
08/09/2017 14:54:29( crit): Object2(setOut): UI(setValue): resolve failed (6521:60000)localhost
08/09/2017 14:54:29( info): Object1(make_child_object): create Object3...
08/09/2017 14:54:29( crit): Object3(setdata): UI(setValue): resolve failed (6618:60000)localhost
08/09/2017 14:54:29( crit): Object3(setOut): UI(setValue): resolve failed (6618:60000)localhost
08/09/2017 14:54:29( crit): Object3(setdata): UI(setValue): resolve failed (6619:60000)localhost
08/09/2017 14:54:29( crit): Object3(setOut): UI(setValue): resolve failed (6619:60000)localhost
08/09/2017 14:54:29( crit): Object3(setdata): UI(setValue): resolve failed (6621:60000)localhost
08/09/2017 14:54:29( crit): Object3(setOut): UI(setValue): resolve failed (6621:60000)localhost
08/09/2017 14:54:29( info): Object1(make_child_object): create Control1...
08/09/2017 14:54:29( info): Object1(make_child_object): create Object1...
08/09/2017 14:54:29( crit): Object4(setdata): UI(setValue): resolve failed (6912:60000)localhost
08/09/2017 14:54:29( crit): Object4(setOut): UI(setValue): resolve failed (6912:60000)localhost
08/09/2017 14:54:29( crit): Object4(setdata): UI(setValue): resolve failed (6913:60000)localhost
08/09/2017 14:54:29( crit): Object4(setOut): UI(setValue): resolve failed (6913:60000)localhost
08/09/2017 14:54:29( crit): Object4(setdata): UI(setValue): resolve failed (6915:60000)localhost
08/09/2017 14:54:29( crit): Object4(setOut): UI(setValue): resolve failed (6915:60000)localhost
08/09/2017 14:54:29( info): Object1(make_child_object): create Object2...
08/09/2017 14:54:29( info): Object1(make_child_object): create Object5...
08/09/2017 14:54:29( info): Object1(make_child_object): create Object6...
08/09/2017 14:54:29( init): (main): -------------- process start -------------------------
\ No newline at end of file
uniset2.files
View file @ dbfe112f
......@@ -31,6 +31,7 @@ extensions/LogDB/LogDB.cc
extensions/LogDB/LogDB.h
extensions/LogDB/LogDBSugar.h
extensions/LogDB/main.cc
extensions/LogDB/logdb-conv.cc
extensions/LogDB/Makefile.am
extensions/include/Calibration.h
extensions/include/ComediInterface.h
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment