I’ve got the same issue (0 usage) but all the data for yesterday is visible on the website; tad stumped?
Config below, I’ve removed references to the forced reading to prevent the errors as seen by others above:
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
import appdaemon.plugins.hass.hassapi as hass
import time
import json
import ctypes
import datetime
#####################
# App configuration #
#####################
# Complete with your settings
anglia_username = "#####"
anglia_password = "#####"
usagesensor = "sensor.water_usage_x"
###################
# WATER USAGE APP #
###################
webpage = r"https://my.anglianwater.co.uk/"
class GetWaterUsage(hass.Hass):
# Starting APP #
def initialize(self):
self.log("Starting Water Usage App")
runtime = datetime.time(20, 00, 00)
zero = datetime.time(00, 00, 0)
self.run_daily(self.StartProcess, runtime)
self.run_daily(self.EndProcess, zero)
#Callback Function to Start the process
def StartProcess(self, kwargs):
self.log("starting process")
usage = self.GetUsage()
self.log("Usage: " + usage)
try:
self.set_state(usagesensor, state = usage)
except Exception as e:
self.log("Set Sensor State Error: " + str(e))
#Callback Function to Zero the readings
def EndProcess(self, kwargs):
self.log("zero readings")
usage = "0.0"
self.log("Usage: " + usage)
self.set_state(usagesensor, state = usage)
# Get Water Usage from Anglia Water website #
def GetUsage(self):
usage = "0.0"
try:
self.log("starting request")
service = Service(executable_path=r'/usr/bin/chromedriver')
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--disable-dev-shm-usage')
browser = webdriver.Chrome(service=service, options=chrome_options)
browser.set_window_size(1600, 1200)
try:
self.log("logging in")
browser.get(webpage)
sbox = browser.find_element(By.ID,'existUser')
sbox.send_keys(anglia_username)
sbox = browser.find_element(By.ID,'existPass')
sbox.send_keys(anglia_password)
button = browser.find_element(By.ID,'existingLogIn')
button.click()
time.sleep(5)
self.log("selecting usage")
button = browser.find_element(By.ID,'btnViewUsage')
button.click()
time.sleep(5)
self.log("saving usage")
html = browser.page_source
browser.quit()
startdata = html.find("var myUsageDetails_days = ") + 26
enddata = html.find(";;", startdata)
result = html[startdata: enddata]
data = json.loads(result.replace("'", '"'))
json_body = []
for x in data:
item = x["usage"] * 1000
json_body.append(item)
usage = str(json_body[-1])
except Exception as e:
self.log("Request Error: " + str(e))
browser.close()
except Exception as e:
self.log("GetUsage Error: " + str(e))
return usage
They’ve been on the webpage each time before 8pm; I’ve left this running over the weekend and have some data in HA, but only starting Sat. Friday didn’t read.
So it seems your sensor.water_usage_x is being updated by the script as it’s supposed to, no? Although, I can see that Friday readings are missing, I have noticed on a few occasions readings from the day before are not available at the same time as always, and it does leave me with a gap in my data which I have to update manually, probably due to some maintenance Anglia waters is doing on their website, it’s very random and I am not savvy enough to go work on a workaround.
However, I can also see that you have two Water sensors, one that I assume is the Utility meter (“Water”) and one that is called “Water Usage”, the 1st should be created on the back of the “water usage x” one but not sure why you created the latter for? Can you just confirm if the Utility meter has been set up as per instructions on my first post, against the “water usage x”? As it’s not showing any readings on any of the days and it should, considering you do have values on the “water usage x” sensor.
Hi,
Pretty sure I did this per the guide, I’m guessing I must have slipped up or misunderstood somewhere to end up creating the extra sensor. I’ve switched the meter to look at the working sensor and will troubleshoot from there.
I think the extra sensor was being created by this:
sensor.water_usage_x:
state_class: total_increasing
sensor.water_usage:
device_class: water
unit_of_measurement: 'L'
I’ve instead used:
sensor.water_usage_x:
state_class: total_increasing
device_class: water
unit_of_measurement: "L"
Maybe I also added the helper separately or something; either way, seems to be running fine now using the hourly script having removed the extra sensor.
Thanks!
Ah. @Zeunas, just a heads up, but I think they’ve changed the website substantially this morning.
Format is different and I have a css element not found error.
Bah, thank you for the heads-up…I was expecting that with the “use the new login to your account” banner they had for quite some time… Don’t think I will be able to work on this during the week but will deff look this weekend.
Just had some time to work on it and so far I can only login and go to the daily usage page, I am trying to figure out how can I extract previous date data but it my be beyond my capabilities…will deff need some time to work on this but won’t put all of my hopes on me working it out
Have some bad news guys…it seems Anglia Waters are now blocking scraping their website…I have been trying to work on my script and bump into an error that seems to block it from login in.
I’m afraid I will have to abandon this, until someone can find a workaround. I will leave this post here just in case but this to say, if you do want to continue having readings on your HA, you will need to add it manually via some sensor or just editing the meter reader’s statistics.
I had this email from Anglian Water the other day, it seems they’re aware that people are scraping their website, they haven’t explicitly said it’s not allowed
“I’m contacting you because we’ve recently noticed some unusual activity on your online account at my.anglianwater.co.uk. To briefly explain this, it looks like your account is being logged into at hourly intervals every day, and this has been happening since around the 22 October. You may already be aware of this, for example, if you’ve set up an automated script to read your smart meter data from the usage page, however, we want to make sure this is you and that you’re aware of what’s happening.”
Thanks for sharing! That adds up now to what we are experiencing with the new login form. I suppose since they introduced the new smart meter there’s been tones of customers scraping their website which my have caused some issues their end…we’ll it’s been fun guys.
It’s not completely dead in the water, they’ve come back to my email yesterday where I asked them for an official api, they’ve said
“Currently there are no plans to offer an API, however this is something we have had several requests for. Once we have fully moved over to the new platform and we start looking at enhancements, this will certainly be something that is discussed, and hopefully we can provide something.”
Ive currently got a working version, once ive tested it will post final version on here.
The reason for the change is that Anglian Water have moved to a new Website developed using AngualarJS, Selenium does not play well with this and we have to incorporate more wait times for pages to load (Javascript DOM).
This is the version that works at the moment… need a few more days testing.
UPDATED!
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from datetime import datetime as date,timedelta
from dateutil import parser
import appdaemon.plugins.hass.hassapi as hass
import time
import json
import ctypes
import datetime
#####################
# App configuration #
#####################
# Complete with your settings
anglia_username = "_YOUR_ANGLIA_WATER_EMAIL-"
anglia_password = "_YOUR_ANGLIA_WATER_PASSWORD_"
usagesensor = "sensor.water_usage_x"
# Optional: Uncomment the below line in case you want to have a switch to run the script on demand if for eg. the script doesn't run at 11am because your HA is restarting. Make sure your input_boolean matches the below:
#force_reading = "input_boolean.water_reading"
###################
# WATER USAGE APP #
###################
webpage = "https://www.anglianwater.co.uk/"
class GetWaterUsage(hass.Hass):
# Starting APP #
def initialize(self):
self.log("Starting Water Usage App")
runtime = datetime.time(11, 00, 0)
zero = datetime.time(00, 00, 0)
self.run_daily(self.StartProcess, runtime)
self.run_daily(self.EndProcess, zero)
self.listen_state(self.StartProcessForced,force_reading, attribute="state", new="on", duration=10 )
#Callback Function to Start the process
def StartProcess(self, kwargs):
self.log("starting process")
usage = self.GetUsage()
self.log("Usage: " + usage)
try:
self.set_state(usagesensor, state = usage)
except Exception as e:
self.log("Set Sensor State Error: " + str(e))
#FORCED Callback Function to Start the process
def StartProcessForced(self, entity, attribute, old, new, kwargs):
self.log("starting process")
self.turn_off(force_reading)
usage = self.GetUsage()
self.log("Usage: " + usage)
try:
self.set_state(usagesensor, state = usage)
except Exception as e:
self.log("Set Sensor State Error: " + str(e))
#Callback Function to Zero the readings
def EndProcess(self, kwargs):
self.log("zero readings")
usage = "0.0"
self.log("Usage: " + usage)
self.set_state(usagesensor, state = usage)
# Get Water Usage from Anglia Water website #
def GetUsage(self):
usage = "0.0"
try:
self.log("starting request")
service = Service(executable_path=r'/usr/bin/chromedriver')
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--disable-dev-shm-usage')
browser = webdriver.Chrome(service=service, options=chrome_options)
browser.set_window_size(1600, 1200)
try:
self.log("Request Home page")
browser.get(webpage)
time.sleep(5)
self.log("Requested Home Page")
browser.find_element(By.LINK_TEXT, "Log In").click()
time.sleep(5)
self.log("Request Login")
browser.find_element(By.ID, "email").send_keys(anglia_username)
browser.find_element(By.ID,'password').send_keys(anglia_password)
browser.find_element(By.ID, "next").click()
time.sleep(10)
self.log("Login")
browser.find_element(By.CSS_SELECTOR, ".justify-content-between:nth-child(3) > .fw-bold").click()
time.sleep(10)
self.log("Request Usage")
ele = browser.find_elements(By.TAG_NAME, 'path')
#Daily
usage = 0
lastreadingdate = date.today().strftime("%d-%b-%Y")
for x in ele:
val = x.get_attribute("aria-label")
if val is not None:
dss = val.split()
stringdt = parser.parse(dss[0] + '-' + dss[1]+ '-' + dss[2]).strftime("%d-%b-%Y")
curdt = (date.today() - timedelta(days=1)).strftime("%d-%b-%Y")
if stringdt == curdt:
usage = dss[3]
lastreadingdate = stringdt
browser.quit()
self.log("Browser quit")
except Exception as e:
self.log("Request Error: " + str(e))
browser.close()
except Exception as e:
self.log("GetUsage Error: " + str(e))
return usage
Hi @yatik I’m getting the following error when I try your updated script:
2024-02-12 17:00:00.013486 INFO WaterUsage: starting process
2024-02-12 17:00:00.015359 INFO WaterUsage: starting request
2024-02-12 17:00:00.548166 INFO WaterUsage: Request Home page
2024-02-12 17:00:07.569364 INFO WaterUsage: Requested Home Page
2024-02-12 17:00:07.605873 INFO WaterUsage: Request Error: Message: no such element: Unable to locate element: {“method”:“link text”,“selector”:“Log In”}
(Session info: chrome-headless-shell=121.0.6167.85)
Hopefully something simple? Thanks for all your work on this @Zeunas & @yatik
Ive updated the code and it was a typo on my part.
Ive been running it for the last 3 days, it seems to update between 8pm-10pm.
Seems to be working reliably so can say this is final version below.
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from datetime import datetime as date,timedelta
from dateutil import parser
import appdaemon.plugins.hass.hassapi as hass
import time
import json
import ctypes
import datetime
#####################
# App configuration #
#####################
# Complete with your settings
anglia_username = "_YOUR_ANGLIA_WATER_EMAIL-"
anglia_password = "_YOUR_ANGLIA_WATER_PASSWORD_"
usagesensor = "sensor.water_usage_x"
# Optional: Uncomment the below line in case you want to have a switch to run the script on demand if for eg. the script doesn't run at 11am because your HA is restarting. Make sure your input_boolean matches the below:
#force_reading = "input_boolean.water_reading"
###################
# WATER USAGE APP #
###################
webpage = "https://www.anglianwater.co.uk/"
class GetWaterUsage(hass.Hass):
# Starting APP #
def initialize(self):
self.log("Starting Water Usage App")
runtime = datetime.time(22, 00, 0)
zero = datetime.time(00, 00, 0)
self.run_daily(self.StartProcess, runtime)
self.run_daily(self.EndProcess, zero)
self.listen_state(self.StartProcessForced,force_reading, attribute="state", new="on", duration=10 )
#Callback Function to Start the process
def StartProcess(self, kwargs):
self.log("starting process")
usage = self.GetUsage()
self.log("Usage: " + usage)
try:
self.set_state(usagesensor, state = usage)
except Exception as e:
self.log("Set Sensor State Error: " + str(e))
#FORCED Callback Function to Start the process
def StartProcessForced(self, entity, attribute, old, new, kwargs):
self.log("starting process")
self.turn_off(force_reading)
usage = self.GetUsage()
self.log("Usage: " + usage)
try:
self.set_state(usagesensor, state = usage)
except Exception as e:
self.log("Set Sensor State Error: " + str(e))
#Callback Function to Zero the readings
def EndProcess(self, kwargs):
self.log("zero readings")
usage = "0.0"
self.log("Usage: " + usage)
self.set_state(usagesensor, state = usage)
# Get Water Usage from Anglia Water website #
def GetUsage(self):
usage = "0.0"
try:
self.log("starting request")
service = Service(executable_path=r'/usr/bin/chromedriver')
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--disable-dev-shm-usage')
browser = webdriver.Chrome(service=service, options=chrome_options)
browser.set_window_size(1600, 1200)
try:
self.log("Request Home page")
browser.get(webpage)
time.sleep(5)
self.log("Requested Home Page")
browser.find_element(By.LINK_TEXT, "Log In").click()
time.sleep(8)
self.log("Request Login")
browser.find_element(By.ID, "email").send_keys(anglia_username)
browser.find_element(By.ID,'password').send_keys(anglia_password)
browser.find_element(By.ID, "next").click()
time.sleep(10)
self.log("Login")
browser.find_element(By.CSS_SELECTOR, ".justify-content-between:nth-child(3) > .fw-bold").click()
time.sleep(10)
self.log("Request Usage")
ele = browser.find_elements(By.TAG_NAME, 'path')
#Daily
usage = 0
lastreadingdate = date.today().strftime("%d-%b-%Y")
for x in ele:
val = x.get_attribute("aria-label")
if val is not None:
dss = val.split()
stringdt = parser.parse(dss[0] + '-' + dss[1]+ '-' + dss[2]).strftime("%d-%b-%Y")
curdt = (date.today() - timedelta(days=1)).strftime("%d-%b-%Y")
if stringdt == curdt:
usage = dss[3]
lastreadingdate = stringdt
browser.quit()
self.log("Browser quit")
except Exception as e:
self.log("Request Error: " + str(e))
browser.close()
except Exception as e:
self.log("GetUsage Error: " + str(e))
return usage
