Is there some trick I can use so the lowpass filter remembers where it left off after a restart?
Not the worst example in this screenshot but I think it does illustrate my point. Blue is the lowpass filter, restarts on the red arrows:
No, there is no way you can restore the previous samples the low pass filter was using before the restart.
You could make a feature request to have them restored along with the state.
1 Like
Hello,
did you find a workaround for this problem ?
Thanks.
Not really. I mostly stopped using this component instead.
I’m serious with Home Assistant since only few days (christmas time) but I think it’s really impossible to do something serious in automation with it nativelly (and I’m a professionnal in automation, it’s not only the hassle to be event based and not state based). A simple thing like IIR 1st order low pass filter is impossible to do.
The one provided is definitively not a IIR 1st order Low Pass at all. It’s not time based so if sensor rate change filter change (and I believe most, if not all, sensor don’t use fixed rate, if value change rate increase a lot, if value remain constant quite no event generated). And no, I will not use polling in an event based system.
This plus the problem of reset on restart which make it not usable for long time filter. And IIR Low Pass is really the basic for filtering followed by deadband.
Relatively simple boolean logic seems impossible to do too. OK on event based system it’s always more complicated. I tryed Note-Red and with “Boolean Ultimate” contrib it’s perfectly doable. With native Home Assistant I saw no way to mix AND OR XOR NOT with different event comming, of course, at different time (need to remember the last state based on last event)
So I switch for Node-Red for quite everything. After only one day digging with Node-Red at least I can do what I want. Here is a Low Pass Filter I use. Probably it’s not very clean (as I’m with Node-Red since one day) but at least it works like it should.
// IIR 1st order low pass filter to incoming payload values
var filter = Number(env.get('filter'));
var tc = filter * 1000; // time constant in milliseconds
var newInput = Number(msg.payload);
var now = new Date();
if (filter < 3600) {
var prevTime = context.get('prevTime') || null;
var prevInput = context.get('prevInput');
var prevOutput = context.get('prevOutput');
}
else {
var prevTime = context.get('prevTime', 'persistent') || null;
var prevInput = context.get('prevInput', 'persistent');
var prevOutput = context.get('prevOutput', 'persistent');
}
if (typeof prevInput == "undefined") prevInput = newInput;
if (typeof prevOutput == "undefined") prevOutput = prevInput;
var newOutput = prevOutput;
if (!isNaN(newInput) && isFinite(newInput)) {
if (filter < 3600) context.set('prevInput', newInput);
else context.set('prevInput', newInput, 'persistent');
}
if (prevTime !== null) {
var dt = now.getTime() - prevTime.getTime();
if (dt > 100) {
if (!isNaN(newInput) && isFinite(newInput)) {
if (filter < 3600) context.set('prevTime', now);
else context.set('prevTime', now, 'persistent');
}
if (!isNaN(prevInput) && isFinite(prevInput)) {
var dtotc = dt / tc;
if (dtotc > 1) dtotc = 1;
newOutput = prevOutput * (1 - dtotc) + prevInput * dtotc;
}
}
}
else
if (!isNaN(newInput) && isFinite(newInput)) {
if (filter < 3600) context.set('prevTime', now);
else context.set('prevTime', now, 'persistent');
}
if (!isNaN(newOutput) && isFinite(newOutput)) {
if (filter < 3600) context.set('prevOutput', newOutput);
else context.set('prevOutput', newOutput, 'persistent');
msg.payload = newOutput.toFixed(2);
}
else
msg = null;
return msg;
Note: “persistent” is the name of the storage on file in my Node-Red config. This should be added to settings.js. I switch to persistent if filter time is > 1 hour. When use persistent better to chain 2 filters to limit the wrtite on disk. For example 1st one with 1 min work with data in RAM then the real one with 1 hour work with data saved in file so this one is called at a max rate of 1 time by minute and don’t flood the disk with writtings.
// Persistent storage
contextStorage: {
default: {
module: "memory"
},
persistent: {
module: "localfilesystem"
}
}
};
for info complete sub-flow I use to filter all data I sent to recorder :
The 1st trigger add one additional event for sensor send no event anymore when value equals 0 to avoid filtered value stay stucked to something <> 0.
[
{
"id": "25133b210a1ba9c5",
"type": "subflow",
"name": "Clean",
"info": "",
"category": "",
"in": [
{
"x": 60,
"y": 80,
"wires": [
{
"id": "2160e9d28309a93b"
},
{
"id": "19a6a6fc3aacf74a"
}
]
}
],
"out": [
{
"x": 1100,
"y": 80,
"wires": [
{
"id": "e6bab6a7ae644b9b",
"port": 0
}
]
}
],
"env": [
{
"name": "deadband",
"type": "str",
"value": "1"
},
{
"name": "filter",
"type": "str",
"value": "60"
}
],
"meta": {},
"color": "#DDAA99"
},
{
"id": "288c2e028ea41f53",
"type": "rbe",
"z": "25133b210a1ba9c5",
"name": "",
"func": "deadbandEq",
"gap": "${deadband}",
"start": "",
"inout": "out",
"septopics": true,
"property": "payload",
"topi": "topic",
"x": 600,
"y": 80,
"wires": [
[
"e6a6ff361ea19988"
]
]
},
{
"id": "d83ace7c44e4410b",
"type": "delay",
"z": "25133b210a1ba9c5",
"name": "",
"pauseType": "rate",
"timeout": "5",
"timeoutUnits": "seconds",
"rate": "1",
"nbRateUnits": "1",
"rateUnits": "hour",
"randomFirst": "1",
"randomLast": "5",
"randomUnits": "seconds",
"drop": true,
"allowrate": false,
"outputs": 1,
"x": 610,
"y": 160,
"wires": [
[
"e6bab6a7ae644b9b"
]
]
},
{
"id": "e6a6ff361ea19988",
"type": "delay",
"z": "25133b210a1ba9c5",
"name": "",
"pauseType": "rate",
"timeout": "5",
"timeoutUnits": "seconds",
"rate": "1",
"nbRateUnits": "1",
"rateUnits": "minute",
"randomFirst": "1",
"randomLast": "5",
"randomUnits": "seconds",
"drop": true,
"allowrate": false,
"outputs": 1,
"x": 790,
"y": 80,
"wires": [
[
"e6bab6a7ae644b9b"
]
]
},
{
"id": "e6bab6a7ae644b9b",
"type": "rbe",
"z": "25133b210a1ba9c5",
"name": "",
"func": "rbe",
"gap": "1%",
"start": "",
"inout": "out",
"septopics": true,
"property": "payload",
"topi": "topic",
"x": 970,
"y": 80,
"wires": [
[]
]
},
{
"id": "19a6a6fc3aacf74a",
"type": "function",
"z": "25133b210a1ba9c5",
"name": "Low Pass",
"func": "// IIR 1st order low pass filter to incoming payload values\n\nvar filter = Number(env.get('filter'));\nvar tc = filter * 1000; // time constant in milliseconds\n\nvar newInput = Number(msg.payload);\nvar now = new Date();\n\nif (filter < 3600) {\n var prevTime = context.get('prevTime') || null;\n var prevInput = context.get('prevInput');\n var prevOutput = context.get('prevOutput');\n}\nelse {\n var prevTime = context.get('prevTime', 'persistent') || null;\n var prevInput = context.get('prevInput', 'persistent');\n var prevOutput = context.get('prevOutput', 'persistent');\n}\n\nif (typeof prevInput == \"undefined\") prevInput = newInput;\nif (typeof prevOutput == \"undefined\") prevOutput = prevInput;\n\nvar newOutput = prevOutput;\n\nif (!isNaN(newInput) && isFinite(newInput)) {\n if (filter < 3600) context.set('prevInput', newInput);\n else context.set('prevInput', newInput, 'persistent');\n}\n\nif (prevTime !== null) {\n\n var dt = now.getTime() - prevTime.getTime();\n\n if (dt > 100) {\n\n if (!isNaN(newInput) && isFinite(newInput)) {\n if (filter < 3600) context.set('prevTime', now);\n else context.set('prevTime', now, 'persistent');\n }\n\n if (!isNaN(prevInput) && isFinite(prevInput)) {\n var dtotc = dt / tc;\n if (dtotc > 1) dtotc = 1;\n newOutput = prevOutput * (1 - dtotc) + prevInput * dtotc;\n }\n }\n}\nelse\n if (!isNaN(newInput) && isFinite(newInput)) {\n if (filter < 3600) context.set('prevTime', now);\n else context.set('prevTime', now, 'persistent');\n }\n\nif (!isNaN(newOutput) && isFinite(newOutput)) {\n if (filter < 3600) context.set('prevOutput', newOutput);\n else context.set('prevOutput', newOutput, 'persistent');\n msg.payload = newOutput.toFixed(2);\n}\nelse\n msg = null;\n\nreturn msg;\n",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 420,
"y": 80,
"wires": [
[
"288c2e028ea41f53",
"d83ace7c44e4410b",
"f737bb0321bf1aaf"
]
]
},
{
"id": "2160e9d28309a93b",
"type": "trigger",
"z": "25133b210a1ba9c5",
"name": "",
"op1": "",
"op2": "",
"op1type": "nul",
"op2type": "payl",
"duration": "${filter}",
"extend": true,
"overrideDelay": false,
"units": "s",
"reset": "",
"bytopic": "all",
"topic": "topic",
"outputs": 1,
"x": 220,
"y": 120,
"wires": [
[
"19a6a6fc3aacf74a"
]
]
},
{
"id": "f737bb0321bf1aaf",
"type": "trigger",
"z": "25133b210a1ba9c5",
"name": "",
"op1": "",
"op2": "",
"op1type": "nul",
"op2type": "payl",
"duration": "${filter}",
"extend": true,
"overrideDelay": false,
"units": "s",
"reset": "",
"bytopic": "all",
"topic": "topic",
"outputs": 1,
"x": 620,
"y": 120,
"wires": [
[
"e6bab6a7ae644b9b"
]
]
}
]
Note : to respect Shannon theorem filter must be > 19s (60/pi) in theory and 60s in real life if max flow rate is 1 by minute.