Read RS232 transmission using SoftwareSerial in esp8266

[raszevsky (Luke)]

Hi, I am trying to integrate the central heating furnace controller in my home with Home Assistant. The furnace controller transmits using some kind of RS protocol. It transmits full frames containing sensor headers and their corresponding data. These frames consist of something about 200 characters, which with the help of esp8266 are collected into String for further processing. I tried using the examples (Custom UART Device, UART Bus) from the esphome website importing my code into them, I also tried using the ready-made example https://esphome.io/cookbook/uart_text_sensor.html#custom-uart-text-sensor but none of the above ways worked for me.
The problem with which I am asking you for help is to transfer the code written in ArduinoIDE to ESPHome (for flashing the esp and return the resulting string to the home assistant as a sensor (in the future, several sensors).
Schematic of connections between esp and the furnace:

arduino code that I want to translate to esphome:

#include <SoftwareSerial.h>

int rx_pin = 5;
int tx_pin = 4;
SoftwareSerial COSerial(rx_pin, tx_pin); // RX, TX

void setup() {
  pinMode (rx_pin, INPUT_PULLUP); 
  Serial.begin(9600);
  while (!Serial);
  Serial.println("HW Serial - Ready");

  COSerial.begin(9600);
  while (!COSerial);
  Serial.println("SW Serial - Ready");

}

String readSerial() {
  int inChar;
  String inStr = "";
  char buff[2];
  long startTime = millis();

  if (COSerial.available()) {

    while (millis() - startTime < 300) {
      inChar = -1;
      inChar = COSerial.read();
      if (inChar > -1) {
      
        sprintf(buff,"%02X",inChar);
        inStr = inStr + buff;
      }
    }
  }
  return inStr;
}

void loop() {

  String fromSerial = readSerial();

  if (fromSerial.length() > 0) {

    Serial.println("Data from CO:");
    Serial.println(fromSerial);
    Serial.println("=============");
        
  }

}

I would be grateful for any hint

[nickrout]

@raszevsky

none of the above ways worked for me.

No yaml, no logs, hard to help.

[raszevsky (Luke)]

@nickrout You’re right, my bad. I’m a little confused about what I’m trying to do here. For now, I managed to transfer the previous code from arduino and import it into esphome. The code looks like this:
yaml:

esphome:
  name: co-overdrive
  includes:
    - piec-uart.h


esp8266:
  board: esp01_1m

# Enable logging
logger:
  level: VERBOSE #makes uart stream available in esphome logstream
  baud_rate: 0 #disable logging over uart
# Enable Home Assistant API
api:
  encryption:
    key: "xxx"

ota:
  password: "xxx"

wifi:
  ssid: !secret wifi_ssid
  password: !secret wifi_password

  # Enable fallback hotspot (captive portal) in case wifi connection fails
  ap:
    ssid: "xxx"
    password: "xxx"

captive_portal:
    
uart:
  id: uart_bus
  tx_pin: 4
  rx_pin: 5
  baud_rate: 9600
  debug:
    direction: RX
    dummy_receiver: false
    after:
      delimiter: "\n"

sensor:
- platform: custom
  lambda: |-
    auto heating_control = new coSensor(id(uart_bus));
    App.register_component(heating_control);
    return {
      heating_control -> COtemp_sensor
    };
  sensors:
    name: "COtemp Sensor"
- platform: custom
  lambda: |-
    auto heating_control = new coSensor(id(uart_bus));
    App.register_component(heating_control);
    return {
      heating_control -> CWUtemp_sensor
    };
  sensors:
    name: "CWUtemp Sensor"
- platform: custom
  lambda: |-
    auto heating_control = new coSensor(id(uart_bus));
    App.register_component(heating_control);
    return {
      heating_control -> PODtemp_sensor
    };
  sensors:
    name: "PODtemp Sensor"
- platform: custom
  lambda: |-
    auto heating_control = new coSensor(id(uart_bus));
    App.register_component(heating_control);
    return {
      heating_control -> gotStr_sensor
    };
  sensors:
    name: "CO RS transmission"

custom component piec-uart.h:

#include "esphome.h"

class coSensor : public PollingComponent, public UARTDevice, public Sensor   {
 public:
 
  // constructor - where we “initialize” the custom sensor
  coSensor(UARTComponent *parent) : UARTDevice(parent), PollingComponent(5000){}
  
  Sensor *COtemp_sensor    = new Sensor();
  Sensor *CWUtemp_sensor  = new Sensor();
  Sensor *PODtemp_sensor        = new Sensor();
  //Sensor *gotStr_sensor        = new Sensor();
  //Sensor *PODstate_sensor        = new Sensor();
  //Sensor *PUMPSmode_sensor        = new Sensor();
  //Sensor *coPUMPstate_sensor        = new Sensor();
  //Sensor *cwuPUMPstate_sensor        = new Sensor();
  //Sensor *WORKmode_sensor    = new Sensor();  

  void setup() override {
    pinMode (5, INPUT_PULLUP);
    // This will be called by App.setup()
  }
  String readSerial(){
    int inChar;
    String inStr = "";
    String finalStr = "";
    char buff[2];
    long startTime = millis();

    if (available()) {

      while (millis() - startTime < 300) {
        inChar = -1;
        inChar = read();
        if (inChar > -1) {
        
          sprintf(buff,"%02X",inChar);
          inStr = inStr + buff;
        }
      }
      if(inStr.length() > 190){
        finalStr = inStr;
      }
    
    }
    return finalStr;
  }
  void update() override {
    // This will be called every "update_interval" milliseconds.

    float COtemp;
    float CWUtemp;
    float PODtemp;
    float PODstate;
    float PUMPSmode;    
    //float gotStr;

    String line = readSerial();
    if(line.length() > 200){
    
        
        COtemp   = line.substring(38,41).toFloat();
        CWUtemp = line.substring(54,57).toFloat();
        PODtemp       = line.substring(142,145).toFloat();
        //PODstate       = line.substring(21,28).toFloat();
        //PUMPSmode   = line.substring(29,34).toFloat();

        COtemp_sensor      -> publish_state(COtemp / 10);
        CWUtemp_sensor    -> publish_state(CWUtemp / 10);
        PODtemp_sensor          -> publish_state(PODtemp / 10);
        //PODstate_sensor          -> publish_state(PODstate);
        //PUMPSmode_sensor      -> publish_state(PUMPSmode);
    }
    else{
        //Does nothing
    }
  }
};

some logs from the esp:

[18:42:42][V][sensor:076]: 'COtemp Sensor': Received new state 6.100000
[18:42:42][D][sensor:126]: 'COtemp Sensor': Sending state 6.10000  with 0 decimals of accuracy
[18:42:42][V][sensor:076]: '': Received new state 28.100000
[18:42:42][D][sensor:126]: '': Sending state 28.10000  with 0 decimals of accuracy
[18:42:42][V][sensor:076]: '': Received new state 0.100000
[18:42:42][D][sensor:126]: '': Sending state 0.10000  with 0 decimals of accuracy
[18:42:42][V][component:200]: Component <unknown> took a long time for an operation (0.35 s).
[18:42:42][V][component:201]: Components should block for at most 20-30ms.
[18:42:42][D][uart_debug:114]: <<< 00:06:02:18:E1:85
[18:42:47][E][uart:015]: Reading from UART timed out at byte 0!
[18:42:47][E][uart:015]: Reading from UART timed out at byte 0!
[18:42:47][E][uart:015]: Reading from UART timed out at byte 0!
[18:42:47][V][sensor:076]: 'COtemp Sensor': Received new state 6.100000
[18:42:47][D][sensor:126]: 'COtemp Sensor': Sending state 6.10000  with 0 decimals of accuracy
[18:42:47][V][sensor:076]: '': Received new state 28.100000
[18:42:47][D][sensor:126]: '': Sending state 28.10000  with 0 decimals of accuracy
[18:42:47][V][sensor:076]: '': Received new state 0.100000
[18:42:47][D][sensor:126]: '': Sending state 0.10000  with 0 decimals of accuracy
[18:42:47][V][component:200]: Component <unknown> took a long time for an operation (0.33 s).
[18:42:47][V][component:201]: Components should block for at most 20-30ms.
[18:42:47][D][uart_debug:114]: <<< 02:26:FF:FA:16:EF:00:01:16:C2:00:00:15:A7:00:09:16:FF:00:06:16:9E:50:32:16:9F:4B:28:15:7F:00:00:15:7D:02:BC:15:7E:00:3C:16:6E:02:BC:16:16:00:3C:16:81:FC:18:15:7C:00:50:15:CD:00:03:16:20:09:14:16:21:00:01:15:89:00:00:15:8B:00:00:01:F6:00:00:02:FC:00:00:02:8E:00:00:01:F9:00:00:02:98:00:00:02:99:00:00:02:45:00:00:03:12:00:00:03:11:00:00:16:81:F8:30:02:18:C5:75
[18:42:48][V][component:200]: Component <unknown> took a long time for an operation (0.30 s).
[18:42:48][V][component:201]: Components should block for at most 20-30ms.
[18:42:48][D][uart_debug:114]: <<< 02:26:FF:F4:16:9E:50:32:16:9F:4B:28:15:7C:00:50:15:7D:02:BC:15:7E:00:3C:16:6E:02:BC:16:16:00:3C:16:81:F8:30:15:CD:00:03:16:20:09:14:16:21:00:01:15:89:00:00:15:8B:00:00:15:87:00:00:15:88:00:00:15:9B:00:00:16:F8:01:86:16
[18:42:52][E][uart:015]: Reading from UART timed out at byte 0!
[18:42:52][E][uart:015]: Reading from UART timed out at byte 0!
[18:42:52][V][sensor:076]: 'COtemp Sensor': Received new state 2.400000
[18:42:52][D][sensor:126]: 'COtemp Sensor': Sending state 2.40000  with 0 decimals of accuracy
[18:42:52][V][sensor:076]: '': Received new state 2.200000
[18:42:52][D][sensor:126]: '': Sending state 2.20000  with 0 decimals of accuracy
[18:42:52][V][sensor:076]: '': Received new state 16.100000
[18:42:52][D][sensor:126]: '': Sending state 16.10000  with 0 decimals of accuracy
[18:42:52][V][component:200]: Component <unknown> took a long time for an operation (0.31 s).
[18:42:52][V][component:201]: Components should block for at most 20-30ms.
[18:42:52][D][uart_debug:114]: <<< 84:00:00:01:F6:00:00:02:8E:00:00:02:98:00:00:02:99:00:00:02:45:00:00:02:18:67:58:02:26:FF:FA:16:EF:00:02:16:C2:00:00:15:A7:00:09:16:FF:00:06:16:9E:50:32:16:9F:4B:28:15:7F:00:00:15:7D:02:BC:15:7E:00:3C:16:6E:02:BC:16:16:00:3C:16:81:FC:18:15:7C:00:50:15:CD:00:03:16:20:09:14:16:21:00:01:15:89:00:00:15:8B:00:00:01:F6:00:00:02:FC:00:00:02:8E:00:00:01:F9:00:00:02:98:00:00:02:99:00:00:02:45:00:00:03:12:00:00:03:11:00:00:16:81:F8:30:02:18:A1:C3
[18:42:57][E][uart:015]: Reading from UART timed out at byte 0!
[18:42:57][V][component:200]: Component <unknown> took a long time for an operation (0.30 s).
[18:42:57][V][component:201]: Components should block for at most 20-30ms.
[18:42:57][V][component:200]: Component <unknown> took a long time for an operation (0.30 s).
[18:42:57][V][component:201]: Components should block for at most 20-30ms.
[18:42:58][V][component:200]: Component <unknown> took a long time for an operation (0.31 s).
[18:42:58][V][component:201]: Components should block for at most 20-30ms.

And at this point I’m stuck. The data returned by the sensors is for now to be ignored (because the transmission is not read correctly and the returned values are complete nonsense). The uart_debug logs displayed in the esphome logs, I believe, are not returning values received by my imported code. Is there a way to output the contents of this string in esphome logs?
The entire correct HEX frame I was before able to receive (using the arduino code posted in the original post, without esphome’s involvement, just reading the logs from esp via serial) looks more or less like this:
0226FFFA16EF000116C2000015A7000916FF0006169E5032169F4B28157F0000157D02C6157E003C166E02C61616003C1681FC18157C005015CD0003162003051621000015890000158B000001F6000002FC0000028E000001F9000002980000029900000245000003120000031100001681F83002181214
Which can be then decoded as in the screenshot below (sorry for polish language):

I know that the correctness of this transmission can be verified using crc-16, but I can’t quite even implement that in the code in esp. I would also like to collect only those frames that have a fixed length and the order of the data that they contain (so that they can then be decoded from HEX to DEC and returned as values for the sensors in HomeAssistant.
To summarize: what I am trying to achieve here is reading the raw transmission from the UART, checking the correctness of the read data (crc, length of the collected string, order of the data in it) and then sending specific parameters as sensor values to the HA.

PS.
Frankly, I wonder if it wouldn’t be easier for me to build my own (for example, based on a combination of an ATmel and esp8266/32) controller to handle central heating and simply replace the current one (also based on some ATMega`s μC) with it.

[nickrout]

There is a post somewhere on this forum detailing how to send uart input to logs and then read the logs. I’ll see if I can find it.

[raszevsky (Luke)]

I’ve tried using the ESP_LOGD("main", "[main] Data collected from UART: ", line); , where the “line” is the String structure containing UART data, but it only prints the "[main] Data collected from UART: " text and nothing else. I’m not sure what type exactly this “String” structure is

[mulcmu]

I think this is the post @nickrout was looking to reference for you.

[How to] UART read without custom component

The UART Bus component provides a write action but no read actions. Typically the UART read is implemented in a custom component that also does the processing. The UART debug sequence can be exploited to act like a call back function whenever there is received data to process. So basically instead of creating a small custom component a large lambda call can be used in the UART debug yaml. This is shared to hopefully help make implementing UART devices a bit easier for average users. It seems t…

[raszevsky (Luke)]

Thanks, it helped me a lot and simplified things. One problem remained, in the hex array received from the UART there are numeric values, encoded in HEX precisely. And thus, for example, I have bytes[18] = 02 and bytes[19] = DA which together give 02DA which converted to decimal, gives me 730 (divided by 10 gives a temperature value of 73 degrees celsius).
Is there a simple way to add/concatenate these two elements of the bytes[] array, convert them to a decimal and return them as a temperature value in sensor?
I found something similar to my problem: https://community.home-assistant.io/t/hex-string-to-dec-value/458945?u=raszevsky
but idk how to implement it in lambda in sensor.

[mulcmu]

Yeah, something along these lines:

ESP_LOGD("BLE TPMS", "  temp: %f",  (x[10] | (x[11] << 8) ) / 100.0 );

Make sure to use 10.0 so you get a float value.

[raszevsky (Luke)]

That works flawlessly! Can I ask one more thing, how to return this calculation in template sensor?
tried few things in yaml, but with no success
yaml:

uart:
  id: uart_bus
  tx_pin: 1
  rx_pin: 3
  baud_rate: 9600
  debug:
    direction: RX
    dummy_receiver: true
    #after:
      #delimiter: [0xCC, 0x33]
    sequence:
      - lambda: |-
          //UARTDebug::log_hex(direction, bytes, '-' );  //Still log the data
          
          if (bytes.size()==100) {
            if(bytes[0]==0x02 && bytes[3]==0xF4) {
              //Pompa CO
              if(bytes[51]==0x00){
                id(COpumpState).publish_state("Wył");}
              if(bytes[51]==0x01){
                id(COpumpState).publish_state("Wł");}
              //Pompa CWU
              if(bytes[55]==0x00){
                id(CWUpumpState).publish_state("Wył");}
              if(bytes[55]==0x01){
                id(CWUpumpState).publish_state("Wł");}
              //Podajnik
              if(bytes[59]==0x00){
                id(PODstate).publish_state("Wył");}
              if(bytes[59]==0x01){
                id(PODstate).publish_state("Wł");}
              //Tryb pomp
              if(bytes[39]==0x00){
                id(Pumpmode).publish_state("Ogrzewanie domu");}
              if(bytes[39]==0x02){
                id(Pumpmode).publish_state("Pompy równoległe");}
              if(bytes[39]==0x03){
                id(Pumpmode).publish_state("Tryb letni");}
              if(bytes[39]==0x04){
                id(Pumpmode).publish_state("Priorytet bojlera");}
              //float COtemp = bytes[19] | (bytes[18] << 8) / 10.0;
              //id(tempCO).publish_state(COtemp);}
              ESP_LOGD("temperatura CO", "  temp: %X",  bytes[18]);
              ESP_LOGD("temperatura CO", "  temp: %X",  bytes[19]);
              ESP_LOGD("temperatura CO", "  temp: %f",  (bytes[19] | (bytes[18] << 8) ) / 10.0);
            }   
          }
       

text_sensor:
  - platform: template
    name: "Pompa CO"
    id: "COpumpState"
    icon: "mdi:pump"
  - platform: template
    name: "Pompa CWU"
    id: "CWUpumpState"
    icon: "mdi:pump"
  - platform: template
    name: "Podajnik"
    id: "PODstate"
    icon: "mdi:filter-cog"
  - platform: template
    name: "Tryb pracy"
    id: "Pumpmode"
    icon: "mdi:cog"


binary_sensor:
  - platform: template
    name: "Temperatura CO"
    id: tempCO
    lambda: |-
      return (id(uart_bus).bytes[19] | (id(uart_bus).bytes[18] << 8) ) / 10.0);
  - platform: template
    name: "Temperatura CWU"
    id: tempCWU
  - platform: template
    name: "Temperatura podajnika"
    id: tempPOD

[mulcmu]

These lines you commented out look correct to me. Did you have id(tempCO) in a a regular sensor: block?

@raszevsky

              //float COtemp = bytes[19] | (bytes[18] << 8) / 10.0;
              //id(tempCO).publish_state(COtemp);}

sensor:
  - platform: template
    name: "temperatura CO"
    id: "tempCO"
    icon: mdi:thermometer"

[raszevsky (Luke)]

yep, i have id: tempCO
the problem is in float COtemp = bytes[19] | (bytes[18] << 8) / 10.0; line
compilation errors:

INFO Reading configuration /config/esphome/co-overdrive.yaml...
INFO Generating C++ source...
INFO Compiling app...
Processing co-overdrive (board: esp01_1m; framework: arduino; platform: platformio/espressif8266 @ 3.2.0)
--------------------------------------------------------------------------------
HARDWARE: ESP8266 80MHz, 80KB RAM, 1MB Flash
LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf
Dependency Graph
|-- ESPAsyncTCP-esphome @ 1.2.3
|-- ESPAsyncWebServer-esphome @ 2.1.0
|   |-- ESPAsyncTCP-esphome @ 1.2.3
|   |-- Hash @ 1.0
|   |-- ESP8266WiFi @ 1.0
|-- DNSServer @ 1.1.1
|-- ESP8266WiFi @ 1.0
|-- ESP8266mDNS @ 1.2
|-- noise-c @ 0.1.4
|   |-- libsodium @ 1.10018.1
Compiling /data/co-overdrive/.pioenvs/co-overdrive/src/main.cpp.o
/config/esphome/co-overdrive.yaml: In lambda function:
/config/esphome/co-overdrive.yaml:81:36: error: invalid operands of types '__gnu_cxx::__alloc_traits<std::allocator<unsigned char>, unsigned char>::value_type' {aka 'unsigned char'} and 'double' to binary 'operator|'
   81 |               float COtemp = bytes[19] | (bytes[18] << 8) / 10.0;
/config/esphome/co-overdrive.yaml: In function 'void setup()':
/config/esphome/co-overdrive.yaml:87:8: error: expected ')' before '}' token
   87 |           }
      |        ^
      |        )
   88 | 
      |         
src/main.cpp:526:77: note: to match this '('
  526 |   lambdaaction = new LambdaAction<uart::UARTDirection, std::vector<uint8_t>>([=](uart::UARTDirection direction, std::vector<uint8_t> bytes) -> void {
      |                                                                             ^
/config/esphome/co-overdrive.yaml:88:3: error: no matching function for call to 'esphome::LambdaAction<esphome::uart::UARTDirection, std::vector<unsigned char, std::allocator<unsigned char> > >::LambdaAction()'
   88 | 
      |   ^

it works in the ESP_LOGD message, but not outside it

[raszevsky (Luke)]

whatever, I made a typo in id(tempCO).publish_state(COtemp);}, one } bracket too far
Nooow it works like a charm, thanks for help.

[adamuss (Adam)]

Witam. Chciałem sie dowiedzieć skąd ma pan taki zrzut ekranu z opisami sterownika kotła? Pytam bo chciałem zrobić to samo co pan tylko ze sterownikiem ST-7110 pid.

[skapron (Szymon)]

@adamuss Here you go: https://dzien.net/tech/index.php?baza=slownik
@raszevsky Can you post here your latest working code?