Is it possible to create a custom filter (moving z-score)?

Hi

I have some applications that I’d like to convert to the Esphome platform. It seems more useful than every time writing a bunch of custom code. But I am still in the learning curve.

Is it possible to create a custom filter? Perhaps this is possible with a custom component a bit like the Resistance Sensor using the output of another one. But perhaps it is also possible to make only an additional filter?

I have an outlier detection based on a moving z-score algorithm that I would like to implement in esphome.
For those interested (not my post): algorithm - Peak signal detection in realtime timeseries data - Stack Overflow

If you can write it in C you can use a lambda filter, https://esphome.io/components/sensor/index.html#lambda

Hi

Thx for the response. I saw the lambda functions, but I cannot figure out how to put my code into 1 single lambda function.
Below the code as a custom sensor:

project.yaml

esp32:
  board: nodemcu-32s
  framework:
    type: arduino
esphome:
  name: project01
  build_path: "./build"
logger:
sensor:
  - platform: dht
    model: DHT11
    pin: 25
    update_interval: 5s
    temperature:
      id: temperature
    humidity:
      name: vochtigheid
      id: humidity
  - platform: ldo_peakdetect
    id: peakdetect
    sensor: humidity
    algorithm: NORMAL
    lag: 10
    threshold: 2
    influence: 0.2
    epsilon: 0.02

sensor.py

import esphome.codegen as cg
import esphome.config_validation as cv
from esphome.components import sensor
from esphome.const import (
    CONF_SENSOR,
    STATE_CLASS_MEASUREMENT,
    ICON_FLASH,
    CONF_ID,
)

ldo_peakdetect_ns = cg.esphome_ns.namespace("ldo_peakdetect")
LdoPeakDetect = ldo_peakdetect_ns.class_("LdoPeakDetect", cg.Component, sensor.Sensor)

CONF_ALGORITHM = "algorithm"
CONF_LAG = "lag"
CONF_THRESHOLD = "threshold"
CONF_INFLUENCE = "influence"
CONF_EPSILON = "epsilon"

LdoPeakDetectAlgo = ldo_peakdetect_ns.enum("LdoPeakDetectAlgo")
ALGORITHM = {
    "COMPARE": LdoPeakDetectAlgo.COMPARE,
    "NORMAL": LdoPeakDetectAlgo.NORMAL,
}

CONFIG_SCHEMA = (
    sensor.sensor_schema(
        state_class=STATE_CLASS_MEASUREMENT,
    )
    .extend(
        {
            cv.GenerateID(): cv.declare_id(LdoPeakDetect),
            cv.Required(CONF_SENSOR): cv.use_id(sensor.Sensor),
            cv.Required(CONF_ALGORITHM): cv.enum(ALGORITHM, upper=True),
            cv.Required(CONF_LAG): cv.int_,
            cv.Required(CONF_THRESHOLD): cv.int_,
            cv.Required(CONF_INFLUENCE): cv.float_,
            cv.Required(CONF_EPSILON): cv.float_,
        }
    )
    .extend(cv.COMPONENT_SCHEMA)
)

async def to_code(config):
    var = cg.new_Pvariable(config[CONF_ID])
    await cg.register_component(var, config)
    await sensor.register_sensor(var, config)

    sen = await cg.get_variable(config[CONF_SENSOR])
    cg.add(var.set_sensor(sen))
    cg.add(var.set_algorithm(config[CONF_ALGORITHM]))
    cg.add(var.set_threshold(config[CONF_THRESHOLD]))
    cg.add(var.set_lag(config[CONF_LAG]))
    cg.add(var.set_influence(config[CONF_INFLUENCE]))
    cg.add(var.set_epsilon(config[CONF_EPSILON]))

ldo_peakdetect.h

/*******************************************************************************
 * ESPhome custom component: ldo_peakdetect
 * V0.9 LDos 16/02/22: initial version
 *
 * use smoothed Z-score algorithm
 * see: https://stackoverflow.com/questions/22583391/peak-signal-detection-in-realtime-timeseries-data/
 *******************************************************************************/
#pragma once
// ======================================================================
// INCLUDES
#include "esphome/core/component.h"
#include "esphome/components/sensor/sensor.h"
// ======================================================================
// NAMESPACES OPEN
namespace esphome {
namespace ldo_peakdetect {
// ======================================================================
// GLOBALS
enum LdoPeakDetectAlgo {
  COMPARE,
  NORMAL,
};
// ======================================================================
// DEFINE CLASS
class LdoPeakDetect : public Component, public sensor::Sensor {
// =========
public:
// ============================================================
// INITIALISATION
    LdoPeakDetect() {}
// destructor
    ~LdoPeakDetect () {
        delete data_;
        delete avg_;
        delete std_;
    }
// ============================================================
// PUBLIC METHODS
    void set_sensor(Sensor *sensor) { sensor_ = sensor; }
    void setup() override;
    //   bool algorithm: true = common algorithm; false: adapted: only compare deviation to threshold
    void set_algorithm(LdoPeakDetectAlgo algorithm) {
        algorithm_ = algorithm;
    }
    //   int lag: size of window to include past signals (e.g. 30-50)
    void set_lag(int lag);
    //   int threshold: if new value is more then threshold*(stdev) away from avg: give signal (e.g. 2-5)
    void set_threshold(int threshold) {
        threshold_ = threshold;
    }
    //   float influence: z-score at which the algorithm signals; 0: no influence from previous signals  to 1: most influence (e.g. 0.6)
    void set_influence(float influence) {
        influence_ = influence;
    }
    //   float epsilon: minimal stdev to detect e.g. 0.05
    void set_epsilon(float epsilon) {
        epsilon_ = epsilon;
    }
    // initialise counter (mean calculation)
    void resetBuffers ();
    // return status (peak)
    int getStatus ();
    // return status (edge)
    int getEdge ();
    // return last data point filtered by the moving average
    float getFilteredPoint ();
    void dump_config() override;
    float get_setup_priority() const override {
        return setup_priority::DATA;
    }
// =========
protected:
// ============================================================
// PROTECTED VARIABLES
    sensor::Sensor *sensor_;
    LdoPeakDetectAlgo algorithm_;           // mode: false = adapted, true = normal
    int lag_                   = 0;         // smoothed z-score pars
    int threshold_             = 0;         // smoothed z-score pars
    float influence_           = 0.0;       // smoothed z-score pars
    float epsilon_             = 0.0;       // smoothed z-score pars
    // peak detection algo
    int index_                 = 0;         // current index in value buffer
    int status_                = 0;         // convert signal to 0: between, 1: positive pulse; -1: negative pulse
    int status_prev_           = 0;         // status value previous iteration
    int edge_                  = 0;         // status has changed 10: -1 nr 1, 5: 0 nr 1, 1: -1 nr 0, -10: 1 nr -1, -5: 0 nr -1, -1: 1 nr 0,  0: unchanged
    float * data_;                          //
    float * avg_;                           //
    float * std_;                           //
    float deviation_;                       //
    float deviationcmp_;                    //
// ============================================================
// PROTECTED METHODS
    // calculate square mean of (part of) buffer
    float calcSqrMean (int start, int len);
    // calculate average of (part of) buffer
    float calcAvg (int start, int len);
    // calculate standard deviation of (part of) buffer
    float calcStd (int start, int len);
    //
    void process_(float value);
// =========
private:
};
// ======================================================================
// NAMESPACES CLOSE
}  // namespace ldo_peakdetect
}  // namespace esphome

ldo_peakdetect.cpp

/*******************************************************************************
 * ESPhome custom component: ldo_peakdetect
 * V0.9 LDos 16/02/22: initial version
 *
 *******************************************************************************/
// ======================================================================
// INCLUDES
// ======================================================================
#include "ldo_peakdetect.h"
#include "esphome/core/log.h"
// ======================================================================
// NAMESPACES OPEN
namespace esphome {
namespace ldo_peakdetect {
// ======================================================================
// GLOBALS
static const char *const TAG = "LdoPeakDetect";
// ======================================================================
// CLASS
// ============================================================
// PUBLIC METHODS
// SETUP
void LdoPeakDetect::setup() {
  this->sensor_->add_on_state_callback([this](float value) { this->process_(value); });
  if (this->sensor_->has_state())
    this->process_(this->sensor_->state);
}
/*
// LOOP
void LdoSensor01::loop() {
  // no action here
}*/
//
void LdoPeakDetect::dump_config() {
  LOG_SENSOR("", TAG, this);
  ESP_LOGCONFIG(TAG, "  Algorithm: %d", this->algorithm_);
  ESP_LOGCONFIG(TAG, "  Lag:       %d", this->lag_);
  ESP_LOGCONFIG(TAG, "  Threshold: %d", this->threshold_);
  ESP_LOGCONFIG(TAG, "  Influence: %.3f", this->influence_);
  ESP_LOGCONFIG(TAG, "  Epsilon:   %.3f", this->epsilon_);
}
//   int lag: size of window to include past signals (e.g. 30-50)
void LdoPeakDetect::set_lag(int lag) {
    lag_ = lag;
    data_ = new float [lag_ + 1];
    avg_  = new float [lag_ + 1];
    std_  = new float [lag_ + 1];
    resetBuffers();
 }
// initialize values and lists
void LdoPeakDetect::resetBuffers () {
	for (int i=0; i<lag_; ++i) {
		data_[i] = 0.0;
		avg_[i]  = 0.0;
		std_[i]  = 0.0;
	}
	index_ = 0;
	status_ = 0;
	status_prev_ = 0;
	edge_ = false;
}
// return status (peak)
int LdoPeakDetect::getStatus () {
	return status_;
}
// return status (edge)
int LdoPeakDetect::getEdge () {
	return edge_;
}
// return last data point filtered by the moving average
float LdoPeakDetect::getFilteredPoint () {
	int i = index_ % lag_;
	return avg_[i];
}
// ============================================================
// PROTECTED METHODS
// process callback method
void LdoPeakDetect::process_(float value) {
    if (std::isnan(value)) {
        this->publish_state(NAN);
        return;
    }
	status_prev_ = status_;
	status_ = 0;
	int i = index_ % lag_;														// current index
	int j = (index_ + 1) % lag_;												// next index
	deviation_ = value - avg_[i];
	if (algorithm_==LdoPeakDetectAlgo::NORMAL) {
		deviationcmp_ = threshold_ * std_[i];									// "enough" deviation
	} else {
		deviationcmp_ = threshold_;												// "enough" deviation
	}
	if (deviation_ > deviationcmp_) {
		data_[j] = influence_ * value + (1.0 - influence_) * data_[i];			// store value, influenced by previous
		status_ = 1;
	} else if (deviation_ < -deviationcmp_) {
		data_[j] = influence_ * value + (1.0 - influence_) * data_[i];			// store value, influenced by previous
		status_ = -1;
	} else {
		data_[j] = value;														// store value
	}
	avg_[j] = calcAvg(j, lag_);
	std_[j] = calcStd(j, lag_);
	index_++;
	if (index_ >= 16383) { //2^14
		index_ = lag_ + j;
	}
    // set edge detection from current and previous status
	if (status_==1) {
		if (status_prev_==0) {
			edge_ = 5;
		} else if (status_prev_==-1) {
			edge_ = 10;
		} else {
			edge_ = 0;
		}
	} else if (status_==-1) {
		if (status_prev_==0) {
			edge_ = -5;
		} else if (status_prev_==1) {
			edge_ = -10;
		} else {
			edge_ = 0;
		}
	} else {	// status_==0
		if (status_prev_==1) {
			edge_ = -1;
		} else if (status_prev_==-1) {
			edge_ = 1;
		} else {
			edge_ = 0;
		}
	}
    ESP_LOGD(TAG, "'%s' - Result %d", this->name_.c_str(), status_);
    this->publish_state(status_);
}
// calculate square mean
float LdoPeakDetect::calcSqrMean (int start, int len) {
	float value = 0.0;
	for (int i=0; i<len; ++i)
		value += data_[(start + i) % lag_] * data_[(start + i) % lag_];
	return value / len;
}
// calculate average
float LdoPeakDetect::calcAvg (int start, int len) {
	float value = 0.0;
	for (int i=0; i<len; ++i)
		value += data_[(start + i) % lag_];
	return value / len;
}
// calculate standard deviation
float LdoPeakDetect::calcStd (int start, int len) {
	float x1 = calcAvg(start, len);
	float x2 = calcSqrMean(start, len);
	float powx1 = x1 * x1;
	float std = x2 - powx1;
	if (std > -epsilon_ && std < epsilon_) {
		return 0.0;
	} else {
		return sqrt(std);
	}
}
// ======================================================================
// NAMESPACES CLOSE
}  // namespace ldo_peakdetect
}  // namespace esphome