Blocking Call inside event loop

Development
1

Hi guys!
I’m trying to build a small custom component to have my energy usage integrated.
There’s an android app called Eon Smart Control(E.ON Smart Control – Apps bei Google Play) that i use with a separate Lora GW which is connected to my electricity meter.

This is what i have been using in past with additional sensor configuration

#!/usr/bin/python3
# encoding: utf8
#

import requests, json
import os, time
from pathlib import Path
from datetime import datetime
from requests.structures import CaseInsensitiveDict

username = "EMAIL"
password = "PASSWORD"
auth_token_file=".auth-token"
today = "{:%Y-%m-%d}".format(datetime.now())
now = "{:%Y-%m-%dT%H:%M:%S.%f}".format(datetime.now())[:-3]
timeout=5

def file_age(filepath):
    seconds = str(time.time() - os.path.getmtime(filepath))
    return int(seconds.split('.')[0])

def get_token(username, password):
    file_location = Path(auth_token_file)
    if not file_location.is_file() or file_age(auth_token_file) >= 3600:
        url = "https://smartcontrol.eon.de/auth"

        headers = CaseInsensitiveDict()
        headers["Accept"] = "application/json"
        headers["Content-Type"] = "application/json"

        data = '{"username":"%s","password":"%s","method":"login"}' % (username, password)
        response = requests.get(url, headers=headers, data=data, allow_redirects=False, timeout=timeout)
        data=json.loads(response.text)
        return(data['access_token'])
    else:
        token=open(auth_token_file, 'r')
        return(token.readline())

def get_watts(access_token):
    url = "https://api.n2g-iona.net/v2/power/%sT00:00:00.000Z/%sZ" % (today, now)
    headers = CaseInsensitiveDict()
    headers["Accept"] = "application/json"
    headers["Authorization"] = "Bearer %s" % (access_token)
    response = requests.get(url, headers=headers, timeout=timeout)
    data=json.loads(response.text)
    my_dict={'power':data['data']['results'][-1]['power']}
    return(my_dict)

def get_kWh(access_token):
    url = "https://api.n2g-iona.net/v2/meter/info"
    headers = CaseInsensitiveDict()
    headers["Accept"] = "application/json"
    headers["Authorization"] = "Bearer %s" % (access_token)
    response = requests.get(url, headers=headers, timeout=timeout)
    data=json.loads(response.text)
    kWh=int(data['data']['Electricity']['CSD'])
    rounded = round(kWh / 1000)
    my_dict={'kWh': rounded}
    return(my_dict)

def main():
    access_token=get_token(username, password)
    print(get_kWh(access_token))
    print(get_watts(access_token))

if __name__ == '__main__':
    main()

This is what you get when you execute it directly

root@homeassistant:/usr/share/hassio/homeassistant# python3 custom_components/smart_control/smart_control_api.py
{'kWh': 8505}
{'power': 380}

This is my current data structure for the custom component:

root@homeassistant:/usr/share/hassio/homeassistant# tree custom_components/smart_control/
custom_components/smart_control/
|-- __init__.py
|-- __pycache__
|   |-- __init__.cpython-39.pyc
|   |-- sensor.cpython-39.pyc
|   `-- smart_control_api.cpython-39.pyc
|-- manifest.json
|-- sensor.py
`-- smart_control_api.py

this is my current sensor.py

"""Platform for sensor integration."""
from __future__ import annotations

from homeassistant.components.sensor import (
    SensorEntity,
    STATE_CLASS_MEASUREMENT,
    DEVICE_CLASS_ENERGY,
)
from homeassistant.const import ENERGY_KILO_WATT_HOUR
from homeassistant.core import HomeAssistant
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
from .smart_control_api import get_token, get_watts, get_kWh


import requests, json
import os, time
from pathlib import Path
from datetime import datetime
from requests.structures import CaseInsensitiveDict


import logging

LOGGER = logging.getLogger(__name__)
USERNAME  = "EMAIL"
PASSWORD = "PASSWORD"

def setup_platform(
    hass: HomeAssistant,
    config: ConfigType,
    add_entities: AddEntitiesCallback,
    discovery_info: DiscoveryInfoType | None = None
) -> None:
    """Set up the sensor platform."""
    add_entities([
        SmartControlEnergyConsumptionTotalSensor(),
        SmartControlPowerConsumptionSensor()
    ])


class SmartControlEnergyConsumptionTotalSensor(SensorEntity):
    """Representation of a Smart Control Energy Consumption Total Sensor."""

    _attr_name = "Smart Control Energy Consumption Total"
    _attr_unit_of_measurement = ENERGY_KILO_WATT_HOUR
    _attr_device_class = DEVICE_CLASS_ENERGY
    _attr_state_class = STATE_CLASS_MEASUREMENT

    def __init__(self):
        self._attr_native_value = None

    async def async_update(self):
        """Fetch new state data for the sensor."""
        #self._attr_native_value = 1337
        try:
            access_token = get_token(USERNAME, PASSWORD)
            LOGGER.debug("ACCESS_TOKEN", access_token)
            kWh = get_kWh(access_token)
            LOGGER.debug("kWh", kWh)
        except requests.exceptions.ReadTimeout:
            self._attr_native_value = None

class SmartControlPowerConsumptionSensor(SensorEntity):
    """Representation of a Smart Control Power Consumption Sensor."""

    _attr_name = "Smart Control Power Consumption"
    _attr_unit_of_measurement = "W"

    def __init__(self):
        self._attr_native_value = None

    async def async_update(self):
        """Fetch new state data for the sensor."""
        #self._attr_native_value = 69
        try:
            access_token = get_token(USERNAME, PASSWORD)
            watts = get_watts(access_token)
            self._attr_native_value = watts['power']
        except requests.exceptions.ReadTimeout:
            self._attr_native_value = None

So far, i tested the sensor with static values ( see commented lines with self._attr_native_value) and it worked without problems. But now, adding my functions to get the data it doesnt work. As far as i understood, it has to do with requests and async - i’m not that deep into programming and have no knowledge of asynchronous programming at all.

This is, what my logfile looks like:

2023-05-27 17:14:35 WARNING (MainThread) [homeassistant.util.async_] Detected blocking call to putrequest inside the event loop. This is causing stability issues. Please report issue to the custom component author for smart_control doing blocking calls at custom_components/smart_control/smart_control_api.py, line 55: response = requests.get(url, headers=headers, timeout=timeout)
2023-05-27 17:14:35 WARNING (MainThread) [homeassistant.util.async_] Detected blocking call to putrequest inside the event loop. This is causing stability issues. Please report issue to the custom component author for smart_control doing blocking calls at custom_components/smart_control/smart_control_api.py, line 45: response = requests.get(url, headers=headers, timeout=timeout)
2023-05-27 17:14:35 ERROR (MainThread) [homeassistant.helpers.entity] Update for sensor.smart_control_energy_consumption_total fails
Traceback (most recent call last):
  File "/usr/src/homeassistant/homeassistant/helpers/entity.py", line 535, in async_update_ha_state
    await self.async_device_update()
  File "/usr/src/homeassistant/homeassistant/helpers/entity.py", line 744, in async_device_update
    raise exc
  File "/config/custom_components/smart_control/sensor.py", line 59, in async_update
    kWh = get_kWh(access_token)
  File "/config/custom_components/smart_control/smart_control_api.py", line 55, in get_kWh
    response = requests.get(url, headers=headers, timeout=timeout)
  File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 529, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 645, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 440, in send
    resp = conn.urlopen(
  File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 398, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 239, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/local/lib/python3.9/http/client.py", line 1285, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/local/lib/python3.9/http/client.py", line 1296, in _send_request
    self.putrequest(method, url, **skips)
  File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 219, in putrequest
    return _HTTPConnection.putrequest(self, method, url, *args, **kwargs)
  File "/usr/src/homeassistant/homeassistant/util/async_.py", line 173, in protected_loop_func
    check_loop(func, strict=strict)
  File "/usr/src/homeassistant/homeassistant/util/async_.py", line 161, in check_loop
    raise RuntimeError(
RuntimeError: Blocking calls must be done in the executor or a separate thread; Use `await hass.async_add_executor_job()` at custom_components/smart_control/smart_control_api.py, line 55: response = requests.get(url, headers=headers, timeout=timeout)
2023-05-27 17:14:35 ERROR (MainThread) [homeassistant.helpers.entity] Update for sensor.smart_control_power_consumption fails
Traceback (most recent call last):
  File "/usr/src/homeassistant/homeassistant/helpers/entity.py", line 535, in async_update_ha_state
    await self.async_device_update()
  File "/usr/src/homeassistant/homeassistant/helpers/entity.py", line 744, in async_device_update
    raise exc
  File "/config/custom_components/smart_control/sensor.py", line 78, in async_update
    watts = get_watts(access_token)
  File "/config/custom_components/smart_control/smart_control_api.py", line 45, in get_watts
    response = requests.get(url, headers=headers, timeout=timeout)
  File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 529, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 645, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 440, in send
    resp = conn.urlopen(
  File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 398, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 239, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/local/lib/python3.9/http/client.py", line 1285, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/local/lib/python3.9/http/client.py", line 1296, in _send_request
    self.putrequest(method, url, **skips)
  File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 219, in putrequest
    return _HTTPConnection.putrequest(self, method, url, *args, **kwargs)
  File "/usr/src/homeassistant/homeassistant/util/async_.py", line 173, in protected_loop_func
    check_loop(func, strict=strict)
  File "/usr/src/homeassistant/homeassistant/util/async_.py", line 161, in check_loop
    raise RuntimeError(
RuntimeError: Blocking calls must be done in the executor or a separate thread; Use `await hass.async_add_executor_job()` at custom_components/smart_control/smart_control_api.py, line 45: response = requests.get(url, headers=headers, timeout=timeout)
~

Any help is greatly appreciated

BR

2

Where you are calling functions from your smart_control_api, do it like this

KWh = await hass.async_add_executor_job(getkWh, access_token)

Ie name of function is first entry in brackets, all passed variables next seperate by commas.

1 Like
3

I have the same warning and the offending line is this:
with open(CONF_file) as f:
can I replace by this:
with await hass.async_add_executor_job(open, CONF_file) as f:
or there is a better way to do it
Thank you

4

No. File access is a blocking function, so whilst async_add_executor_job will run the open in a thread (so as not to block the event loop), reading the file will also be blocking.

If your function reads a file and returns a value, call the whole function by doing…

val = await hass.async_add_executor_job(CONF_FILE_READING_FUNCTION)

Aternatively, use aiofiles to read your file which is non blocking.

5

Thanks, with aiofiles do we need to add an import line at the beginnig of python file.
I get aiofiles is undefined

6

Yes, you always need to import libraries you are using.

7

so this seem to work

import aiofiles
...
    dev_list = []
    async with aiofiles.open(CONF_file) as f:
        async for line in f:
            dev_list.append(json.loads(line))         
    await f.close()

should I put await in front of line dev_list…

8

So if the fix for this

return await self.pvoutput.status()

just

return await hass.async_add_executor_job(self.pvoutput.status())

9

I’m trying to do a similar thing and I can’t get it to work.

I’m using a python package (that makes Request calls) to get some information and got the “Detected blocking call” error when using that package functions.

I tried calling the package function with a “await hass.async_add_executor_job” but that didn’t remove the error. Anyone with any suggestions?

10

Suggest post link to your code and the error message?

11

I want to use pytube to get a list of all videos in a YouTube Playlist.

I can get the custom component to setup fine, but if the code uses the pytube package then I get the blocking error. Here are the important snippets from the sensors.py file:

async def async_setup_platform(
    hass: HomeAssistant,
    config: ConfigType,
    async_add_entities: Callable,
    discovery_info: Optional[DiscoveryInfoType] = None,
) -> None:
    """Set up the sensor platform."""
    session = async_get_clientsession(hass)
    sensors = [YouTubePlaylist(hass=hass, playlist=config[CONF_PLAYLIST])]
    async_add_entities(sensors, update_before_add=True)



class YouTubePlaylist(Entity):
    def __init__(self, hass, playlist):
        super().__init__()
        self.playlist = playlist
        self.hass = hass
        self._available = True
        self.attrs: Dict[str, Any] = {ATTR_PLAYLIST: playlist}
        self._name = "youtube_shuffler"
        self._state = "Init!"

    async def async_update(self):
        try:
            date_time = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
            self.attrs["Time"] = date_time
            self._state = "Updated"

            #yt_playlist = Playlist(self.playlist)
            yt_playlist = await self.hass.async_add_executor_job(Playlist, self.playlist)
            self.attrs["vid"] = yt_playlist[0]

            self._available = True
        except:
            self._available = False
            _LOGGER.exception("Error setting things up.")

And the important snippet out of the log file:

2024-11-26 23:08:09.295 WARNING (MainThread) [homeassistant.util.loop] Detected blocking call to load_default_certs with args (<ssl.SSLContext object at 0x7f3402dfe450>, <Purpose.SERVER_AUTH: _ASN1Object(nid=129, shortname='serverAuth', longname='TLS Web Server Authentication', oid='1.3.6.1.5.5.7.3.1')>) inside the event loop by custom integration 'youtube_shuffler' at custom_components/youtube_shuffler/sensor.py, line 114: self.attrs["vid"] = yt_playlist[0] (offender: /usr/local/lib/python3.12/ssl.py, line 713: context.load_default_certs(purpose)), please report it to the author of the 'youtube_shuffler' custom integration
For developers, please see https://developers.home-assistant.io/docs/asyncio_blocking_operations/#load_default_certs
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/usr/src/homeassistant/homeassistant/__main__.py", line 223, in <module>
    sys.exit(main())
  File "/usr/src/homeassistant/homeassistant/__main__.py", line 209, in main
    exit_code = runner.run(runtime_conf)
  File "/usr/src/homeassistant/homeassistant/runner.py", line 189, in run
    return loop.run_until_complete(setup_and_run_hass(runtime_config))
  File "/usr/local/lib/python3.12/asyncio/base_events.py", line 674, in run_until_complete
    self.run_forever()
  File "/usr/local/lib/python3.12/asyncio/base_events.py", line 641, in run_forever
    self._run_once()
  File "/usr/local/lib/python3.12/asyncio/base_events.py", line 1990, in _run_once
    handle._run()
  File "/usr/local/lib/python3.12/asyncio/events.py", line 88, in _run
    self._context.run(self._callback, *self._args)
  File "/usr/src/homeassistant/homeassistant/helpers/entity_platform.py", line 728, in _async_add_entity
    await entity.async_device_update(warning=False)
  File "/usr/src/homeassistant/homeassistant/helpers/entity.py", line 1302, in async_device_update
    await self.async_update()
  File "/config/custom_components/youtube_shuffler/sensor.py", line 114, in async_update
    self.attrs["vid"] = yt_playlist[0]

2024-11-26 23:08:09.311 WARNING (MainThread) [homeassistant.util.loop] Detected blocking call to putrequest with args (<http.client.HTTPSConnection object at 0x7f33fc4d0b30>, 'GET', '/playlist?list=PL_Hl-KFLCw93iUwrYFW2R_1htoBin5dbg') inside the event loop by custom integration 'youtube_shuffler' at custom_components/youtube_shuffler/sensor.py, line 114: self.attrs["vid"] = yt_playlist[0] (offender: /usr/local/lib/python3.12/http/client.py, line 1347: self.putrequest(method, url, **skips)), please report it to the author of the 'youtube_shuffler' custom integration
For developers, please see https://developers.home-assistant.io/docs/asyncio_blocking_operations/#putrequest
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/usr/src/homeassistant/homeassistant/__main__.py", line 223, in <module>
    sys.exit(main())
  File "/usr/src/homeassistant/homeassistant/__main__.py", line 209, in main
    exit_code = runner.run(runtime_conf)
  File "/usr/src/homeassistant/homeassistant/runner.py", line 189, in run
    return loop.run_until_complete(setup_and_run_hass(runtime_config))
  File "/usr/local/lib/python3.12/asyncio/base_events.py", line 674, in run_until_complete
    self.run_forever()
  File "/usr/local/lib/python3.12/asyncio/base_events.py", line 641, in run_forever
    self._run_once()
  File "/usr/local/lib/python3.12/asyncio/base_events.py", line 1990, in _run_once
    handle._run()
  File "/usr/local/lib/python3.12/asyncio/events.py", line 88, in _run
    self._context.run(self._callback, *self._args)
  File "/usr/src/homeassistant/homeassistant/helpers/entity_platform.py", line 728, in _async_add_entity
    await entity.async_device_update(warning=False)
  File "/usr/src/homeassistant/homeassistant/helpers/entity.py", line 1302, in async_device_update
    await self.async_update()
  File "/config/custom_components/youtube_shuffler/sensor.py", line 114, in async_update
    self.attrs["vid"] = yt_playlist[0]

2024-11-26 23:08:09.311 ERROR (MainThread) [custom_components.youtube_shuffler.sensor] Error setting things up.
Traceback (most recent call last):
  File "/config/custom_components/youtube_shuffler/sensor.py", line 114, in async_update
    self.attrs["vid"] = yt_playlist[0]
                        ~~~~~~~~~~~^^^
  File "/usr/local/lib/python3.12/site-packages/pytube/contrib/playlist.py", line 309, in __getitem__
    return self.video_urls[i]
           ~~~~~~~~~~~~~~~^^^
  File "/usr/local/lib/python3.12/site-packages/pytube/helpers.py", line 57, in __getitem__
    next_item = next(self.gen)
                ^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/pytube/contrib/playlist.py", line 281, in url_generator
    for page in self._paginate():
  File "/usr/local/lib/python3.12/site-packages/pytube/contrib/playlist.py", line 118, in _paginate
    json.dumps(extract.initial_data(self.html))
                                    ^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/pytube/contrib/playlist.py", line 58, in html
    self._html = request.get(self.playlist_url)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/pytube/request.py", line 53, in get
    response = _execute_request(url, headers=extra_headers, timeout=timeout)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/pytube/request.py", line 37, in _execute_request
    return urlopen(request, timeout=timeout)  # nosec
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/urllib/request.py", line 215, in urlopen
    return opener.open(url, data, timeout)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/urllib/request.py", line 515, in open
    response = self._open(req, data)
               ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/urllib/request.py", line 532, in _open
    result = self._call_chain(self.handle_open, protocol, protocol +
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/urllib/request.py", line 492, in _call_chain
    result = func(*args)
             ^^^^^^^^^^^
  File "/usr/local/lib/python3.12/urllib/request.py", line 1392, in https_open
    return self.do_open(http.client.HTTPSConnection, req,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/urllib/request.py", line 1344, in do_open
    h.request(req.get_method(), req.selector, req.data, headers,
  File "/usr/local/lib/python3.12/http/client.py", line 1336, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/local/lib/python3.12/http/client.py", line 1347, in _send_request
    self.putrequest(method, url, **skips)
  File "/usr/src/homeassistant/homeassistant/util/loop.py", line 192, in protected_loop_func
    raise_for_blocking_call(
  File "/usr/src/homeassistant/homeassistant/util/loop.py", line 158, in raise_for_blocking_call
    raise RuntimeError(
RuntimeError: Caught blocking call to putrequest with args (<http.client.HTTPSConnection object at 0x7f33fc4d0b30>, 'GET', '/playlist?list=PL_Hl-KFLCw93iUwrYFW2R_1htoBin5dbg') inside the event loop by custom integration 'youtube_shuffler' at custom_components/youtube_shuffler/sensor.py, line 114: self.attrs["vid"] = yt_playlist[0]. (offender: /usr/local/lib/python3.12/http/client.py, line 1347: self.putrequest(method, url, **skips)), please report it to the author of the 'youtube_shuffler' custom integration
For developers, please see https://developers.home-assistant.io/docs/asyncio_blocking_operations/#putrequest

At a loss for how to use this async stuff correctly to fix my problem so any suggestions are appreciated. I don’t know this stuff very well.

12

So, the issue is that (looking at the pytube code)

yt_playlist[0]

is actually doing a blocking call to read the list of videos from your url and return the first entry. This is being run in the loop and not an executor.

Looking at the pytube code, you maybe better running

playlist = await self.hass.async_add_executor_job(yt_playlist.video_urls)
if playlist:
    self.attrs["vid"] = playlist[0]

Which is basically doing the same thing as your original code but can be run in the executor.

1 Like
13

I appreciate the suggestion. I had also noticed that indexing in the array was causing the error and tried other things to get around that before post to no luck.

Tried the suggestion and it didn’t work but to achieve the same idea (that I think you were going for) I figured I would force the list to be copied to a new list and hopefully finish any blocking calls:

playlist = await self.hass.async_add_executor_job(list, yt_playlist.video_urls)

And this worked! I can access the playlist variable now without any issues.

I appreciate the help!! Thank you very much.