/********************************************************************* * * 文 件: SampleTrendAnalysis.cpp * * 版权所有: Shanghai Baosight Software Co., Ltd. * zoufuzhou * changelog: * 2021-04-07: add tag_name related query to this class *********************************************************************/ #include "mix_cc/algorithm/fast_dtw/EuclideanDistance.h" #include "mix_cc/algorithm/fast_dtw/FastDTW.h" #include #include #include #include #include #include #include using namespace fastdtw; using namespace std; extern iDA::Connection cn; using namespace log4cplus; SampleTrendAnalysis::SampleTrendAnalysis(const string &table, const std::string &ruleId, const std::string &rule_name, int judgemode, int archive, double probability, std::string tag_name) { this->Init(table, ruleId, rule_name, judgemode, archive, probability, tag_name); mp_normal = new NormalDistribution(); } void SampleTrendAnalysis::Init(const string &table, const std::string &ruleId, const std::string &rule_name, int judgemode, int archive, double probability, std::string tag_name) { memset(&m_sample.data.abs_diff, 0, sizeof(m_sample.data.abs_diff)); memset(&m_sample.data.normal_cfdi, 0, sizeof(m_sample.data.normal_cfdi)); memset(&m_sample.state, 0, sizeof(m_sample.state)); memset(&this->m_sample.learn.abs_diff, 0, sizeof(this->m_sample.learn.abs_diff)); this->m_sample.tag_name = tag_name; this->m_table = table; this->rule_id_ = ruleId; this->m_rulename = rule_name; this->m_sample.state.judgemode = judgemode; this->m_sample.state.archive = archive; this->m_sample.state.probability = probability; } SampleTrendAnalysis::~SampleTrendAnalysis() { delete mp_normal; } const SampleProperty *SampleTrendAnalysis::GetProperty() { return &m_sample; } int SampleTrendAnalysis::JudgeMode() { return this->m_sample.state.judgemode; } bool SampleTrendAnalysis::HaveSample() { return this->m_sample.state.havesample; } bool SampleTrendAnalysis::IsLearned() { return this->m_sample.state.islearned; } int SampleTrendAnalysis::Read(const string &dbwhere) { LOG d("SampleTrendAnalysis::Read|" + m_rulename, AUTO_CATCH_PID); int ret = -1; switch (this->m_sample.state.judgemode) { case JUDGE_MODE::DIFFERENCE: case JUDGE_MODE::PRCENTAGE: { ret = ReadAbsDiff(dbwhere); } break; case JUDGE_MODE::NORMAL: { vector voutdatas; ret = this->ReadNormal(dbwhere, voutdatas); d.Debug() << "TEST" << std::endl; if (ret == NULL) { mp_normal->ReSet(voutdatas); mp_normal->Analysis(this->m_sample.state.probability, &this->m_sample.data.normal_cfdi); d.Info() << "Normal zone[" << this->m_sample.data.normal_cfdi.up << "," << this->m_sample.data.normal_cfdi.low << "]" << endl; } voutdatas.clear(); } break; case JUDGE_MODE::TREND_FIT: { ret = this->ReadJson(dbwhere); } break; case JUDGE_MODE::POLYFIT: // do nothing, because poly fit is not dealt in this cpp break; default: { d.Error() << "undefined mode:" << this->m_sample.state.judgemode << endl; return 0; } break; } if (ret != NULL) { this->m_sample.state.havesample = false; } else { this->m_sample.state.havesample = true; } return ret; } int SampleTrendAnalysis::Learning(double value, int batch_size) { LOG d("SampleTrendAnalysis::Learning|" + m_rulename, AUTO_CATCH_PID); int ret = 0; if (value == 0 || value < EPS) { d.Warn() << "new learning data=0, throw the data!" << endl; return -1; } d.Info() << "new learning data:" << value << endl; this->m_sample.learn.vstat.push_back(fabs(value)); if (this->m_sample.learn.vstat.size() >= batch_size) { switch (this->m_sample.state.judgemode) { case JUDGE_MODE::DIFFERENCE: case JUDGE_MODE::PRCENTAGE: { ret = this->LearningAbsDiff(); } break; case JUDGE_MODE::NORMAL: { ret = this->LearningAbsDiff(); if (this->m_sample.state.havesample == false) { ret = this->LearningNormal(); } } break; default: { d.Error() << "undefined mode:" << this->m_sample.state.judgemode << endl; return 0; } break; } this->m_sample.learn.vstat.clear(); time_t now = time(0); if (now - this->m_sample.state.lastsave >= this->m_sample.state.archive * 1 * 3600 || // this->m_sample.state.archive * 24 * 3600 || this->m_sample.state.islearned == false) { ret = this->Save(); if (ret == 0) { this->m_sample.state.lastsave = now; } } if (ret != 0) { this->m_sample.state.islearned = false; } else { if (this->m_sample.state.havesample == false) { memcpy(&m_sample.data.abs_diff, &m_sample.learn.abs_diff, sizeof(m_sample.learn.abs_diff)); } this->m_sample.state.islearned = true; this->m_sample.state.havesample = true; } } return 0; } /** * @brief * @param value My Param doc * @param batch_sizeMy Param doc * @return int */ int SampleTrendAnalysis::Learning(vector value, int batch_size) { LOG d("SampleTrendAnalysis::Learning|" + m_rulename, AUTO_CATCH_PID); int ret = 0; d.Debug() << "Trend learn start" << endl; ret = this->LearningTrend(value); d.Debug() << "Trend learn end" << endl; // if (this->m_sample.learn.vstat.size() >= batch_size) { time_t now = time(0); if (now - this->m_sample.state.lastsave >= // this->m_sample.state.archive * 24 * 3600 || this->m_sample.state.archive * 1 * 3600 || this->m_sample.state.islearned == false) { ret = this->Save(); if (ret == NULL) { this->m_sample.state.lastsave = now; } } this->m_sample.learn.vtrend.clear(); if (ret != NULL) { this->m_sample.state.islearned = false; this->m_sample.state.havesample = false; } else { if (this->m_sample.state.havesample == false) { for (int i = 0; i < m_sample.learn.vtrend.size(); i++) { m_sample.data.vtrend[i] = m_sample.learn.vtrend[i]; } } this->m_sample.state.islearned = true; this->m_sample.state.havesample = true; } } return 0; } int SampleTrendAnalysis::Save(void) { if (last_save_ == time(0)) { return -1; } else { last_save_ = time(0); } LOG d("SampleTrendAnalysis::Save|" + m_rulename, AUTO_CATCH_PID); int ret = -1; string sql = ""; if (this->m_sample.state.judgemode == JUDGE_MODE::TREND_FIT) { Json::Value jvalue; Json::FastWriter json_wtitter; if (this->m_sample.learn.vtrend.size() == 0) { return -1; } for (int i = 0; i < this->m_sample.learn.vtrend.size() && i < SAMPLE_COUNT; i++) { double ftmp = this->m_sample.learn.vtrend[i]; ftmp = 0.001 * ((long)(ftmp * 1000)); jvalue["data"][i] = ftmp; } string json = json_wtitter.write(jvalue); jvalue.clear(); sql = "Insert into T_RULE_SAMPLE (RULEID,SAMPLEDATE,TAGNAME,DATAJSON,MODE) " "VALUES ('" + rule_id_ + "',sysdate, '" + this->m_sample.tag_name + "','" + json + "',1)"; } else { if (this->m_sample.learn.abs_diff.avg == 0 || this->m_sample.learn.abs_diff.avg < EPS) { d.Error() << "learn avg is 0" << endl; return -1; } sql = "Insert into T_RULE_SAMPLE (RULEID,SAMPLEDATE,TAGNAME,DATA) VALUES " "('" + rule_id_ + "',sysdate, '" + this->m_sample.tag_name + "','" + std::to_string(this->m_sample.learn.abs_diff.avg) + "')"; } d.Debug() << "sql:" << sql << endl; long count; DBMag::Execute(sql, &count); if (count > 0) { DbStandardDBAX().dbCommit(); ret = 0; } return 0; } int SampleTrendAnalysis::LearningAbsDiff(void) { /** * @brief 注意,这里将平均值直接置为最新值,可能存在问题 * 所以改成了只存储最新值 */ int ret = 0; if (this->m_sample.learn.abs_diff.avg == 0 && !this->m_sample.learn.vstat.empty()) { this->m_sample.learn.abs_diff.avg = this->m_sample.learn.vstat[0]; } if (!this->m_sample.learn.vstat.empty()) this->m_sample.learn.abs_diff.avg = this->m_sample.learn.vstat[0]; /* for (int i = 0; i < this->m_sample.learn.vstat.size(); i++) { this->m_sample.learn.abs_diff.avg += this->m_sample.learn.vstat[i]; this->m_sample.learn.abs_diff.max = this->m_sample.learn.abs_diff.max > this->m_sample.learn.vstat[i] ? this->m_sample.learn.abs_diff.max : this->m_sample.learn.vstat[i]; this->m_sample.learn.abs_diff.min = this->m_sample.learn.abs_diff.min < this->m_sample.learn.vstat[i] ? this->m_sample.learn.abs_diff.min : this->m_sample.learn.vstat[i]; } this->m_sample.learn.abs_diff.avg /= (this->m_sample.learn.vstat.size() + 1); */ return ret; } int SampleTrendAnalysis::LearningNormal(void) { int ret = 0; this->mp_normal->ReSet(this->m_sample.learn.vstat); ret = this->mp_normal->Analysis(this->m_sample.state.probability, &this->m_sample.data.normal_cfdi); return ret; } bool SampleTrendAnalysis::ShouldReportCurve(const vector &value, double dest) { if (CalcDTWValue(value) > 100) { return true; } // do not learn this curve and report else { this->LearningTrend(value); this->Save(); return false; } } double SampleTrendAnalysis::CalcDTWValue(const vector &value) { LOG d("CalcDTW", AUTO_CATCH_PID); if (!m_sample.data.vtrend.empty()) { TimeSeries tsI; for (int i = 0; i < m_sample.data.vtrend.size(); ++i) { tsI.addLast(i, TimeSeriesPoint(&m_sample.data.vtrend[i])); } TimeSeries tsJ; for (int i = 0; i < value.size(); ++i) { tsJ.addLast(i, TimeSeriesPoint(&value[i])); } TimeWarpInfo info = FAST::getWarpInfoBetween(tsI, tsJ, EuclideanDistance()); return info.getDistance(); } d.Debug() << "Value Size = 0" << std::endl; return 0; } double SampleTrendAnalysis::CalcDTWValue(HD3Record *value) { LOG d("CalcDTW", AUTO_CATCH_PID); if (!m_sample.data.vtrend.empty()) { TimeSeries tsI; for (int i = 0; i < m_sample.data.vtrend.size(); ++i) { tsI.addLast(i, TimeSeriesPoint(&m_sample.data.vtrend[i])); } TimeSeries tsJ; for (int i = 0; i < SAMPLE_COUNT; ++i) { double tmp = value[i].NumberValue(); tsJ.addLast(i, TimeSeriesPoint(&tmp)); } TimeWarpInfo info = FAST::getWarpInfoBetween(tsI, tsJ, EuclideanDistance()); return info.getDistance(); } d.Debug() << "Value Size = 0" << std::endl; return 0; } /** * @brief Add Value to learn Data * @param value My Param doc * @return int * @log */ int SampleTrendAnalysis::LearningTrend(vector value) { int ret = 0; this->m_sample.learn.vtrend = value; return ret; } int SampleTrendAnalysis::ReadAbsDiff(const string &dbwhere) { LOG d("SampleTrendAnalysis::ReadAbsDiff|" + m_rulename, AUTO_CATCH_PID); string sql = GetSql(dbwhere); if (sql == "") { return -1; } d.Debug() << "sql:" << sql << endl; iDA::Command cmd; cmd.SetConnection(&cn); cmd.SetCommandText(sql); try { cmd.Execute(); } catch (iDA::Exception &e) { d.Error() << sql << endl; d.Error() << e.ErrMsg() << endl; return (-1); } memset(&this->m_sample.data.abs_diff, 0, sizeof(this->m_sample.data.abs_diff)); int count = 0; while (cmd.FetchNext()) { this->m_sample.data.abs_diff.max = this->m_sample.data.abs_diff.max > cmd.Field(1).AsDouble() ? this->m_sample.data.abs_diff.max : cmd.Field(1).AsDouble(); this->m_sample.data.abs_diff.min = this->m_sample.data.abs_diff.min != 0 && this->m_sample.data.abs_diff.min < cmd.Field(2).AsDouble() ? this->m_sample.data.abs_diff.min : cmd.Field(2).AsDouble(); this->m_sample.data.abs_diff.avg += cmd.Field(3).AsDouble(); count++; // d.Info()<m_sample.data.abs_diff.max<<","<m_sample.data.abs_diff.min<<","<m_sample.data.abs_diff.avg<m_sample.data.abs_diff.avg = this->m_sample.data.abs_diff.avg / count; d.Info() << "sample max,min,avg:" << this->m_sample.data.abs_diff.max << "," << this->m_sample.data.abs_diff.min << "," << this->m_sample.data.abs_diff.avg << endl; return 0; } int SampleTrendAnalysis::ReadNormal(const string &dbwhere, vector &voutdatas) { LOG d("SampleTrendAnalysis::ReadNormal|" + m_rulename, AUTO_CATCH_PID); string sql = GetSql(dbwhere); if (sql == "") { return -1; } d.Debug() << "sql:" << sql << endl; iDA::Command cmd; cmd.SetConnection(&cn); cmd.SetCommandText(sql); try { cmd.Execute(); } catch (iDA::Exception &e) { d.Error() << sql << endl; d.Error() << e.ErrMsg() << endl; return (-1); } d.Info() << "sample datas:"; while (cmd.FetchNext()) { voutdatas.push_back(cmd.Field(1).AsDouble()); d.Info() << cmd.Field(1).AsDouble() << ","; } d.Info() << endl; if (voutdatas.size() < 5) { return -1; } return 0; } int SampleTrendAnalysis::ReadJson(const string &dbwhere) { LOG d("SampleTrendAnalysis::ReadJson|" + m_rulename, AUTO_CATCH_PID); string sql = GetSql(dbwhere); d.Debug() << "sql:" << sql << endl; if (sql == "") { return -1; } iDA::Command cmd; cmd.SetConnection(&cn); cmd.SetCommandText(sql); try { cmd.Execute(); } catch (iDA::Exception &e) { d.Error() << sql << endl; d.Error() << e.ErrMsg() << endl; return (-1); } Json::Value jvalue; Json::Reader jreader; int count = 0; this->m_sample.data.vtrend.resize(SAMPLE_COUNT, 0); while (cmd.FetchNext()) { jvalue.clear(); d.Debug() << cmd.Field(1).AsString() << endl; if (jreader.parse(cmd.Field(1).AsString(), jvalue)) { for (int i = 0; i < jvalue["data"].size() && i < SAMPLE_COUNT; i++) { this->m_sample.data.vtrend[i] += jvalue["data"][i].asDouble(); } count++; } } if (count == 0) { return -1; } d.Debug() << "sample data:"; for (int i = 0; i < this->m_sample.data.vtrend.size() && i < SAMPLE_COUNT; i++) { this->m_sample.data.vtrend[i] /= count; d.Debug() << " " << this->m_sample.data.vtrend[i]; } d.Debug() << endl; return 0; } string SampleTrendAnalysis::GetSql(const string &dbwhere) { LOG d("SampleTrendAnalysis::GetSql|" + m_rulename, AUTO_CATCH_PID); if (dbwhere.length() == 0) return ""; string sql = ""; switch (this->m_sample.state.judgemode) { case JUDGE_MODE::DIFFERENCE: case JUDGE_MODE::PRCENTAGE: { // sql = "select max(data),min(data),avg(data) from " + m_table; sql = "select max(data),min(data),round(avg(data),5) from " + m_table; } break; case JUDGE_MODE::NORMAL: { sql = "select data from " + m_table; } break; case JUDGE_MODE::TREND_FIT: { if (this->m_table == "T_STA_SAMPLE") { sql = "select data from " + m_table; } else { sql = "select datajson from " + m_table; } } break; default: { return ""; } break; } if (this->GetCount(dbwhere) <= 0) { d.Warn() << "can't get sample from db,where:" << dbwhere << endl; string tmpwhere = dbwhere.substr(0, dbwhere.find("and")); // d.Warn()<GetCount(tmpwhere) <= 0) { d.Warn() << "can't get sample from db,where:" << tmpwhere << endl; return ""; } else { sql = sql + " where " + tmpwhere; } } else { sql = sql + " where " + dbwhere; } return sql; } long SampleTrendAnalysis::GetCount(const string &dbwhere) { LOG d("SampleTrendAnalysis::GetCount|" + m_rulename, AUTO_CATCH_PID); if (dbwhere.length() == 0) return -1; string sql = "select count(*) from " + m_table + " where " + dbwhere; iDA::Command cmd; cmd.SetConnection(&cn); cmd.SetCommandText(sql); try { cmd.Execute(); } catch (iDA::Exception &e) { d.Error() << sql << endl; d.Error() << e.ErrMsg() << endl; return (-1); } if (cmd.FetchNext()) { return cmd.Field(1).AsLong(); } else { d.Error() << "No record found" << endl; return 0; } return 0; }