eis/eqpalg/.do_not_use/stat.bk/SampleTrendAnalysis.cpp

/*********************************************************************
 *
 *  文      件:    SampleTrendAnalysis.cpp
 *
 *  版权所有:   Shanghai Baosight Software Co., Ltd.
 *  zoufuzhou
 * changelog:
 * 2021-04-07: add tag_name related query  to this class
 *********************************************************************/

#include "mix_cc/algorithm/fast_dtw/EuclideanDistance.h"
#include "mix_cc/algorithm/fast_dtw/FastDTW.h"
#include <dao/DBMag.h>
#include <dao/DbStandardDBAX.h>
#include <eqpalg/stat_tools/SampleTrendAnalysis.h>
#include <iDA/iDA.h>
#include <json/json.h>
#include <log4cplus/LOG.h>
#include <math.h>

using namespace fastdtw;

using namespace std;
extern iDA::Connection cn;
using namespace log4cplus;

SampleTrendAnalysis::SampleTrendAnalysis(const string &table,
                                         const std::string &ruleId,
                                         const std::string &rule_name,
                                         int judgemode, int archive,
                                         double probability,
                                         std::string tag_name) {
  this->Init(table, ruleId, rule_name, judgemode, archive, probability,
             tag_name);

  mp_normal = new NormalDistribution();
}

void SampleTrendAnalysis::Init(const string &table, const std::string &ruleId,
                               const std::string &rule_name, int judgemode,
                               int archive, double probability,
                               std::string tag_name) {

  memset(&m_sample.data.abs_diff, 0, sizeof(m_sample.data.abs_diff));
  memset(&m_sample.data.normal_cfdi, 0, sizeof(m_sample.data.normal_cfdi));
  memset(&m_sample.state, 0, sizeof(m_sample.state));
  memset(&this->m_sample.learn.abs_diff, 0,
         sizeof(this->m_sample.learn.abs_diff));

  this->m_sample.tag_name = tag_name;
  this->m_table = table;
  this->rule_id_ = ruleId;
  this->m_rulename = rule_name;
  this->m_sample.state.judgemode = judgemode;
  this->m_sample.state.archive = archive;
  this->m_sample.state.probability = probability;
}

SampleTrendAnalysis::~SampleTrendAnalysis() { delete mp_normal; }

const SampleProperty *SampleTrendAnalysis::GetProperty() { return &m_sample; }

int SampleTrendAnalysis::JudgeMode() { return this->m_sample.state.judgemode; }

bool SampleTrendAnalysis::HaveSample() {
  return this->m_sample.state.havesample;
}

bool SampleTrendAnalysis::IsLearned() { return this->m_sample.state.islearned; }

int SampleTrendAnalysis::Read(const string &dbwhere) {
  LOG d("SampleTrendAnalysis::Read|" + m_rulename, AUTO_CATCH_PID);
  int ret = -1;

  switch (this->m_sample.state.judgemode) {
  case JUDGE_MODE::DIFFERENCE:
  case JUDGE_MODE::PRCENTAGE: {
    ret = ReadAbsDiff(dbwhere);
  } break;
  case JUDGE_MODE::NORMAL: {
    vector<double> voutdatas;
    ret = this->ReadNormal(dbwhere, voutdatas);
    d.Debug() << "TEST" << std::endl;
    if (ret == NULL) {
      mp_normal->ReSet(voutdatas);
      mp_normal->Analysis(this->m_sample.state.probability,
                          &this->m_sample.data.normal_cfdi);
      d.Info() << "Normal zone[" << this->m_sample.data.normal_cfdi.up << ","
               << this->m_sample.data.normal_cfdi.low << "]" << endl;
    }
    voutdatas.clear();

  } break;
  case JUDGE_MODE::TREND_FIT: {
    ret = this->ReadJson(dbwhere);
  } break;
  case JUDGE_MODE::POLYFIT:
  // do nothing, because poly fit is not dealt in this cpp
    break;
  default: {
    d.Error() << "undefined mode:" << this->m_sample.state.judgemode << endl;
    return 0;
  } break;
  }

  if (ret != NULL) {
    this->m_sample.state.havesample = false;
  } else {
    this->m_sample.state.havesample = true;
  }

  return ret;
}

int SampleTrendAnalysis::Learning(double value, int batch_size) {
  LOG d("SampleTrendAnalysis::Learning|" + m_rulename, AUTO_CATCH_PID);
  int ret = 0;
  if (value == 0 || value < EPS) {
    d.Warn() << "new learning data=0, throw the data!" << endl;
    return -1;
  }
  d.Info() << "new learning data:" << value << endl;

  this->m_sample.learn.vstat.push_back(fabs(value));

  if (this->m_sample.learn.vstat.size() >= batch_size) {
    switch (this->m_sample.state.judgemode) {
    case JUDGE_MODE::DIFFERENCE:
    case JUDGE_MODE::PRCENTAGE: {
      ret = this->LearningAbsDiff();
    } break;
    case JUDGE_MODE::NORMAL: {
      ret = this->LearningAbsDiff();
      if (this->m_sample.state.havesample == false) {
        ret = this->LearningNormal();
      }
    } break;
    default: {
      d.Error() << "undefined mode:" << this->m_sample.state.judgemode << endl;
      return 0;
    } break;
    }

    this->m_sample.learn.vstat.clear();
    time_t now = time(0);
    if (now - this->m_sample.state.lastsave >=
            this->m_sample.state.archive * 1 * 3600 ||
        //            this->m_sample.state.archive * 24 * 3600 ||
        this->m_sample.state.islearned == false) {
      ret = this->Save();
      if (ret == 0) {
        this->m_sample.state.lastsave = now;
      }
    }

    if (ret != 0) {
      this->m_sample.state.islearned = false;
    } else {
      if (this->m_sample.state.havesample == false) {
        memcpy(&m_sample.data.abs_diff, &m_sample.learn.abs_diff,
               sizeof(m_sample.learn.abs_diff));
      }
      this->m_sample.state.islearned = true;
      this->m_sample.state.havesample = true;
    }
  }

  return 0;
}

/**
 * @brief
 * @param  value            My Param doc
 * @param  batch_sizeMy Param doc
 * @return int
 */
int SampleTrendAnalysis::Learning(vector<double> value, int batch_size) {
  LOG d("SampleTrendAnalysis::Learning|" + m_rulename, AUTO_CATCH_PID);
  int ret = 0;

  d.Debug() << "Trend learn start" << endl;
  ret = this->LearningTrend(value);
  d.Debug() << "Trend learn end" << endl;

  // if (this->m_sample.learn.vstat.size() >= batch_size)
  {
    time_t now = time(0);
    if (now - this->m_sample.state.lastsave >=
            // this->m_sample.state.archive * 24 * 3600 ||
            this->m_sample.state.archive * 1 * 3600 ||
        this->m_sample.state.islearned == false) {
      ret = this->Save();
      if (ret == NULL) {
        this->m_sample.state.lastsave = now;
      }
    }
    this->m_sample.learn.vtrend.clear();

    if (ret != NULL) {
      this->m_sample.state.islearned = false;
      this->m_sample.state.havesample = false;
    } else {
      if (this->m_sample.state.havesample == false) {

        for (int i = 0; i < m_sample.learn.vtrend.size(); i++) {
          m_sample.data.vtrend[i] = m_sample.learn.vtrend[i];
        }
      }
      this->m_sample.state.islearned = true;
      this->m_sample.state.havesample = true;
    }
  }

  return 0;
}

int SampleTrendAnalysis::Save(void) {
  if (last_save_ == time(0)) {
    return -1;
  } else {
    last_save_ = time(0);
  }
  LOG d("SampleTrendAnalysis::Save|" + m_rulename, AUTO_CATCH_PID);
  int ret = -1;
  string sql = "";
  if (this->m_sample.state.judgemode == JUDGE_MODE::TREND_FIT) {

    Json::Value jvalue;
    Json::FastWriter json_wtitter;
    if (this->m_sample.learn.vtrend.size() == 0) {
      return -1;
    }
    for (int i = 0; i < this->m_sample.learn.vtrend.size() && i < SAMPLE_COUNT;
         i++) {
      double ftmp = this->m_sample.learn.vtrend[i];
      ftmp = 0.001 * ((long)(ftmp * 1000));
      jvalue["data"][i] = ftmp;
    }
    string json = json_wtitter.write(jvalue);
    jvalue.clear();
    sql = "Insert into T_RULE_SAMPLE (RULEID,SAMPLEDATE,TAGNAME,DATAJSON,MODE) "
          "VALUES ('" +
          rule_id_ + "',sysdate, '" + this->m_sample.tag_name + "','" + json +
          "',1)";
  } else {
    if (this->m_sample.learn.abs_diff.avg == 0 ||
        this->m_sample.learn.abs_diff.avg < EPS) {
      d.Error() << "learn avg is 0" << endl;
      return -1;
    }
    sql = "Insert into T_RULE_SAMPLE (RULEID,SAMPLEDATE,TAGNAME,DATA) VALUES "
          "('" +
          rule_id_ + "',sysdate, '" + this->m_sample.tag_name + "','" +
          std::to_string(this->m_sample.learn.abs_diff.avg) + "')";
  }
  d.Debug() << "sql:" << sql << endl;
  long count;
  DBMag::Execute(sql, &count);
  if (count > 0) {
    DbStandardDBAX().dbCommit();
    ret = 0;
  }
  return 0;
}

int SampleTrendAnalysis::LearningAbsDiff(void) {
  /**
   * @brief  注意，这里将平均值直接置为最新值，可能存在问题
   * 所以改成了只存储最新值
   */
  int ret = 0;
  if (this->m_sample.learn.abs_diff.avg == 0 &&
      !this->m_sample.learn.vstat.empty()) {
    this->m_sample.learn.abs_diff.avg = this->m_sample.learn.vstat[0];
  }
  if (!this->m_sample.learn.vstat.empty())
    this->m_sample.learn.abs_diff.avg = this->m_sample.learn.vstat[0];
  /*
  for (int i = 0; i < this->m_sample.learn.vstat.size(); i++) {
    this->m_sample.learn.abs_diff.avg += this->m_sample.learn.vstat[i];
    this->m_sample.learn.abs_diff.max =
        this->m_sample.learn.abs_diff.max > this->m_sample.learn.vstat[i]
            ? this->m_sample.learn.abs_diff.max
            : this->m_sample.learn.vstat[i];
    this->m_sample.learn.abs_diff.min =
        this->m_sample.learn.abs_diff.min < this->m_sample.learn.vstat[i]
            ? this->m_sample.learn.abs_diff.min
            : this->m_sample.learn.vstat[i];
  }
  this->m_sample.learn.abs_diff.avg /= (this->m_sample.learn.vstat.size() + 1);
  */
  return ret;
}

int SampleTrendAnalysis::LearningNormal(void) {
  int ret = 0;
  this->mp_normal->ReSet(this->m_sample.learn.vstat);
  ret = this->mp_normal->Analysis(this->m_sample.state.probability,
                                  &this->m_sample.data.normal_cfdi);
  return ret;
}

bool SampleTrendAnalysis::ShouldReportCurve(const vector<double> &value,
                                            double dest) {

  if (CalcDTWValue(value) > 100) {
    return true;
  } // do not learn this curve and report
  else {
    this->LearningTrend(value);
    this->Save();
    return false;
  }
}

double SampleTrendAnalysis::CalcDTWValue(const vector<double> &value) {
  LOG d("CalcDTW", AUTO_CATCH_PID);

  if (!m_sample.data.vtrend.empty()) {
    TimeSeries<double, 1> tsI;
    for (int i = 0; i < m_sample.data.vtrend.size(); ++i) {
      tsI.addLast(i, TimeSeriesPoint<double, 1>(&m_sample.data.vtrend[i]));
    }

    TimeSeries<double, 1> tsJ;
    for (int i = 0; i < value.size(); ++i) {
      tsJ.addLast(i, TimeSeriesPoint<double, 1>(&value[i]));
    }

    TimeWarpInfo<double> info =
        FAST::getWarpInfoBetween(tsI, tsJ, EuclideanDistance());
    return info.getDistance();
  }
  d.Debug() << "Value Size = 0" << std::endl;

  return 0;
}

double SampleTrendAnalysis::CalcDTWValue(HD3Record *value) {
  LOG d("CalcDTW", AUTO_CATCH_PID);

  if (!m_sample.data.vtrend.empty()) {
    TimeSeries<double, 1> tsI;
    for (int i = 0; i < m_sample.data.vtrend.size(); ++i) {
      tsI.addLast(i, TimeSeriesPoint<double, 1>(&m_sample.data.vtrend[i]));
    }

    TimeSeries<double, 1> tsJ;
    for (int i = 0; i < SAMPLE_COUNT; ++i) {
      double tmp = value[i].NumberValue();
      tsJ.addLast(i, TimeSeriesPoint<double, 1>(&tmp));
    }

    TimeWarpInfo<double> info =
        FAST::getWarpInfoBetween(tsI, tsJ, EuclideanDistance());
    return info.getDistance();
  }
  d.Debug() << "Value Size = 0" << std::endl;

  return 0;
}

/**
 * @brief Add Value to learn Data
 * @param  value            My Param doc
 * @return int
 * @log
 */
int SampleTrendAnalysis::LearningTrend(vector<double> value) {
  int ret = 0;
  this->m_sample.learn.vtrend = value;
  return ret;
}

int SampleTrendAnalysis::ReadAbsDiff(const string &dbwhere) {
  LOG d("SampleTrendAnalysis::ReadAbsDiff|" + m_rulename, AUTO_CATCH_PID);
  string sql = GetSql(dbwhere);
  if (sql == "") {
    return -1;
  }
  d.Debug() << "sql:" << sql << endl;

  iDA::Command cmd;
  cmd.SetConnection(&cn);
  cmd.SetCommandText(sql);
  try {
    cmd.Execute();
  } catch (iDA::Exception &e) {
    d.Error() << sql << endl;
    d.Error() << e.ErrMsg() << endl;
    return (-1);
  }

  memset(&this->m_sample.data.abs_diff, 0,
         sizeof(this->m_sample.data.abs_diff));
  int count = 0;
  while (cmd.FetchNext()) {
    this->m_sample.data.abs_diff.max =
        this->m_sample.data.abs_diff.max > cmd.Field(1).AsDouble()
            ? this->m_sample.data.abs_diff.max
            : cmd.Field(1).AsDouble();
    this->m_sample.data.abs_diff.min =
        this->m_sample.data.abs_diff.min != 0 &&
                this->m_sample.data.abs_diff.min < cmd.Field(2).AsDouble()
            ? this->m_sample.data.abs_diff.min
            : cmd.Field(2).AsDouble();
    this->m_sample.data.abs_diff.avg += cmd.Field(3).AsDouble();
    count++;
    //		d.Info()<<cmd.Field(3).AsString()<<endl;

    //		d.Info()<<"sample
    // max,min,avg:"<<this->m_sample.data.abs_diff.max<<","<<this->m_sample.data.abs_diff.min<<","<<this->m_sample.data.abs_diff.avg<<endl;
  }
  this->m_sample.data.abs_diff.avg = this->m_sample.data.abs_diff.avg / count;
  d.Info() << "sample max,min,avg:" << this->m_sample.data.abs_diff.max << ","
           << this->m_sample.data.abs_diff.min << ","
           << this->m_sample.data.abs_diff.avg << endl;

  return 0;
}

int SampleTrendAnalysis::ReadNormal(const string &dbwhere,
                                    vector<double> &voutdatas) {
  LOG d("SampleTrendAnalysis::ReadNormal|" + m_rulename, AUTO_CATCH_PID);
  string sql = GetSql(dbwhere);
  if (sql == "") {
    return -1;
  }
  d.Debug() << "sql:" << sql << endl;

  iDA::Command cmd;
  cmd.SetConnection(&cn);
  cmd.SetCommandText(sql);
  try {
    cmd.Execute();
  } catch (iDA::Exception &e) {
    d.Error() << sql << endl;
    d.Error() << e.ErrMsg() << endl;
    return (-1);
  }

  d.Info() << "sample datas:";
  while (cmd.FetchNext()) {
    voutdatas.push_back(cmd.Field(1).AsDouble());
    d.Info() << cmd.Field(1).AsDouble() << ",";
  }
  d.Info() << endl;

  if (voutdatas.size() < 5) {
    return -1;
  }
  return 0;
}

int SampleTrendAnalysis::ReadJson(const string &dbwhere) {
  LOG d("SampleTrendAnalysis::ReadJson|" + m_rulename, AUTO_CATCH_PID);

  string sql = GetSql(dbwhere);
  d.Debug() << "sql:" << sql << endl;
  if (sql == "") {
    return -1;
  }

  iDA::Command cmd;
  cmd.SetConnection(&cn);
  cmd.SetCommandText(sql);
  try {
    cmd.Execute();
  } catch (iDA::Exception &e) {
    d.Error() << sql << endl;
    d.Error() << e.ErrMsg() << endl;
    return (-1);
  }

  Json::Value jvalue;
  Json::Reader jreader;
  int count = 0;
  this->m_sample.data.vtrend.resize(SAMPLE_COUNT, 0);
  while (cmd.FetchNext()) {
    jvalue.clear();
    d.Debug() << cmd.Field(1).AsString() << endl;
    if (jreader.parse(cmd.Field(1).AsString(), jvalue)) {
      for (int i = 0; i < jvalue["data"].size() && i < SAMPLE_COUNT; i++) {
        this->m_sample.data.vtrend[i] += jvalue["data"][i].asDouble();
      }
      count++;
    }
  }

  if (count == 0) {
    return -1;
  }
  d.Debug() << "sample data:";
  for (int i = 0; i < this->m_sample.data.vtrend.size() && i < SAMPLE_COUNT;
       i++) {
    this->m_sample.data.vtrend[i] /= count;
    d.Debug() << " " << this->m_sample.data.vtrend[i];
  }
  d.Debug() << endl;

  return 0;
}

string SampleTrendAnalysis::GetSql(const string &dbwhere) {
  LOG d("SampleTrendAnalysis::GetSql|" + m_rulename, AUTO_CATCH_PID);
  if (dbwhere.length() == 0)
    return "";

  string sql = "";
  switch (this->m_sample.state.judgemode) {
  case JUDGE_MODE::DIFFERENCE:
  case JUDGE_MODE::PRCENTAGE: {
    //    sql = "select max(data),min(data),avg(data) from " + m_table;
    sql = "select max(data),min(data),round(avg(data),5) from " + m_table;
  } break;
  case JUDGE_MODE::NORMAL: {
    sql = "select data from " + m_table;
  } break;
  case JUDGE_MODE::TREND_FIT: {
    if (this->m_table == "T_STA_SAMPLE") {
      sql = "select data from " + m_table;
    } else {
      sql = "select datajson from " + m_table;
    }
  } break;
  default: {
    return "";
  } break;
  }
  if (this->GetCount(dbwhere) <= 0) {
    d.Warn() << "can't get sample from db,where:" << dbwhere << endl;
    string tmpwhere = dbwhere.substr(0, dbwhere.find("and"));
    //	  d.Warn()<<tmpwhere<<endl;
    tmpwhere = tmpwhere.substr(0, tmpwhere.find("AND"));
    //	  d.Warn()<<tmpwhere<<endl;
    if (this->GetCount(tmpwhere) <= 0) {
      d.Warn() << "can't get sample from db,where:" << tmpwhere << endl;
      return "";
    } else {
      sql = sql + " where " + tmpwhere;
    }
  } else {
    sql = sql + " where " + dbwhere;
  }
  return sql;
}

long SampleTrendAnalysis::GetCount(const string &dbwhere) {
  LOG d("SampleTrendAnalysis::GetCount|" + m_rulename, AUTO_CATCH_PID);
  if (dbwhere.length() == 0)
    return -1;

  string sql = "select count(*) from " + m_table + " where " + dbwhere;

  iDA::Command cmd;
  cmd.SetConnection(&cn);
  cmd.SetCommandText(sql);
  try {
    cmd.Execute();
  } catch (iDA::Exception &e) {
    d.Error() << sql << endl;
    d.Error() << e.ErrMsg() << endl;
    return (-1);
  }

  if (cmd.FetchNext()) {
    return cmd.Field(1).AsLong();
  } else {
    d.Error() << "No record found" << endl;
    return 0;
  }
  return 0;
}