I’ve been stuck for so long now, so i’m asking for help. I’m working on making a scraping sensor using requests_html as the site i’m scraping ha js content. My current working file is a bit of a mess, but I keep trying to change and move functions around to get it to scrape. Initially I tried to do it non-async, i get the html and can view it in the logs, but the command r.html.render(sleep=2) keeps resulting in RuntimeError: There is no current event loop in thread 'SyncWorker_4'. So now I’ve tried to changing it to be async, but it’s still the same line that fails, however now with a different error: RuntimeError: Blocking calls must be done in the executor or a separate thread; Use `await hass.async_add_executor_job()` at custom_components.euronext.sensor.py, line 129: await self._response.html.arender()
This is the code i’m currently working on, it’s a bit of a mess:
##############################
# configuration.yaml example #
##############################
# sensor:
# - platform: euronext
# funds:
# - NO0010582984.DKGLBIX-WOMF
# - NO0010337959.DIUSA-WOMF
# - NO0010582976.DKNORIX-WOMF
from datetime import timedelta
import logging
from requests_html import AsyncHTMLSession
import voluptuous as vol
from homeassistant.components.sensor import PLATFORM_SCHEMA
from homeassistant.const import (
ATTR_ATTRIBUTION,
CONF_SCAN_INTERVAL,
CONF_CURRENCY
)
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity import Entity
_LOGGER = logging.getLogger(__name__)
ATTRIBUTION = "Data provided by Euronext"
CONF_FUNDS = 'funds'
DEFAULT_SCAN_INTERVAL = timedelta(minutes=1)
PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
{
vol.Required(CONF_FUNDS): vol.All(cv.ensure_list),
vol.Optional(CONF_SCAN_INTERVAL, default=DEFAULT_SCAN_INTERVAL): cv.time_period
}
)
# def scrape_data(input):
# try:
# _LOGGER.debug('scraping......')
# session = requests_html.AsyncHTMLSession()
# response = session.get('https://live.euronext.com/en/product/funds/'+input)
# _LOGGER.debug(response.html.html)
# response.html.render(sleep=2)
# return response, session
# except AttributeError:
# print('AttributeError for %s, please check configuration', input)
async def async_setup_platform(hass, config, async_add_entities, discovery_info=None):
'''Setup the Euronext sensor. '''
_LOGGER.debug('setting up......')
funds = config.get(CONF_FUNDS)
unit_of_measurement = config.get(CONF_CURRENCY, 'kr')
_LOGGER.info("List of funds: %s", funds)
for fund in funds:
_LOGGER.info("for-loop fund: %s", fund)
try:
_LOGGER.info('Configuring fund %s', fund)
if ' ' not in fund:
# session = AsyncHTMLSession()
async_add_entities([EuronextSensor(fund, unit_of_measurement)], True)
else:
_LOGGER.error('Values for "fund:" can not contain spaces, found "%s"', fund)
except ValueError:
_LOGGER.error('Error loading fund %s, please check config', fund)
_LOGGER.info('Setup of funds complete')
class EuronextSensor(Entity):
'''Representation of the Euronext Fond sensor.'''
def __init__(self, fund, unit_of_measurement):
self._fund = fund
self._asession = None
self._state = None
self._unique_id = None
self._name = ""
self._icon = "mdi:timer-sand-empty"
self._unit_of_measurement = unit_of_measurement
self._attrs = {ATTR_ATTRIBUTION: ATTRIBUTION}
# self._response = None
@property
def name(self):
'''Return the name of the sensor.'''
return self._name
@property
def state(self):
'''Return the state of the device.'''
return self._state
@property
def unique_id(self):
'''Return the unique ID.'''
return self._unique_id
@property
def unit_of_measurement(self):
'''Return the unit of measurement of this entity.'''
return self._unit_of_measurement
@property
def state_attributes(self):
'''Return the state attributes.'''
return self._attrs
@property
def icon(self):
'''Return icon to use based on preformance.'''
return self._icon
async def async_update(self):
# _LOGGER.debug('updating......')
_LOGGER.info('Requesting new data for %s', self._fund)
_LOGGER.debug('scraping......')
self._asession = AsyncHTMLSession()
self._response = await self._asession.get('https://live.euronext.com/en/product/funds/'+self._fund)
# _LOGGER.debug(self._response.html.html)
# _LOGGER.debug('scraping......')
# response = session.get('https://live.euronext.com/en/product/funds/'+input)
# _LOGGER.debug(self._response.html.html)
await self._response.html.arender(sleep=2)
# self._response = scrape_data(self._fund)
# _LOGGER.info('Data updated for fund %s (%s)', self._fund, self._response.html.find('h1', first=True).text)
self._name = self._response.html.find('h1', first=True).text
_LOGGER.debug(self._name)
self._state = self._response.html.find('#header-instrument-price', first=True).text.replace(',', '.').replace(' ', '')
self._unique_id = self._response.html.find('.enx-symbol-top-custom', first=True).text
date = self._response.html.find('#fs_fund_nav_block tr:nth-of-type(3) td:nth-of-type(3)', first=True).text
historical = self._response.html.find('#fs_fund_historical_prices_block table', first=True)
historical = [[td.text for td in tr.find('td')[:-2]] for tr in historical.find('tr')][2:]
self._attrs = {item[0]: item[1].replace(',', '.') + ' %' for item in historical if item[1] != '-'}
self._attrs['Dato'] = f'{date[8:10]}.{date[5:7]}.{date[0:4]}'
iconval = float(response.html.find('.data-24', first=True).text.replace(',', '.'))
self._icon = 'mdi:trending-up' if iconval > 0 else 'mdi:trending-down' if iconval < 0 else 'mdi:trending-neutral'