Scraping Voltaware meter data?

I want to poll a Voltaware monitor that’s measuring power consumption in my house. I can use a browser to view the data direct from the monitor (image below). I thought I could “just” use an “http request” node in Node-RED and parse the html UTF-8 response. However, the payload is gobbledegook - example below? What am I doing wrong?

��&�b�meterdata.html�VMs�6��W �E� �t�C’9�q�ine;�LO�%�����_�)%�d�b���[�vW�W������ZTdt~6���L�oB�4柝&X�Gqo���AB?w�� �����% ��ߒ���Y�T����DHg -CW��*+q�$���PV�� �_���4f���a�P�S� �� ��R 5{��1�O3Ԅ�ϟ��U�0ie�G�%-0T�|���<K�B�2���_��P�&��eI��G6����a�UM"xYe�닯!�'�n�8��K�q�%���*�x6w�<��\g��2KjX�,~�η�+�R(�*@>,�kl9S&�*�K���!e�4���܉)�D��h�AR�Xv���� |֮t��q�[��ʀ����.�ȝ͒�zIJ��2U��6˔]�?ٕ7��w{Hؼ�� qO����f�j�3R��ԫE� p�J+���I~�M�pć�%>^�:��r�=i���V{��������a��Sܤ�~b%Zy�t�/�}/� h�} ���V^�̃�/y�8Q��m�����B�Dž�w�k��-Z�2��%\�m�A<��Y+^y�K�[����_�;�e�eҧ�[]k��W��u���k�e�o3�L���ߒ6��;K��:���B�[u��fT�#���.u�al��i9���4x��N�>��nt�x%7�`4�/�|�˻!���7Kqm�/�b4=g���0�2����we#�a��!�KIQ�[�/C��v�F"
Q�Wj�B)q����=r;�����;F<�!����P,�a�f,’��.�k޽Ͻ�1w��5�C^���݀�e�s��ņ�3

It’s probably not UTF 8 then.
Try another format and see if anyone works, or open the source and see what the source says the format is

Turns out the device sends the page as compressed data - found a gzip node and added it to the Node-RED flow and I can see html for the page now.

Next problem seems to be that the html paints a table which is populated by some javascript - how do I get hold of the data ?

You can’t do that in a simple way. Not sure if it’s even possible at all with node red/HA

Had a rummage around in the javascript and there was another URL that returns the data in json format.

I can now poll the Voltaware monitor and get “live” power consumption data - just what I wanted.

Ciao. Hai risolto ? Hai integrato in home assistant ? Puoi darmi qualche indicazione ? Grazie

Hi,

I use Node-RED to build the workflows. The key discovery was that the Voltaware monitor produces a webpage with all the data in a json object. Find the URL for your Voltaware device and the webpage is “/getmeterdata” - so for me it’s “192.168.1.229/getmeterdata”.

In node-RED I have an:

  • “inject” node to create a one minute polling interval
  • “http request” node that GETs the data from the Voltaware device
  • “json” parser node that converts the json object into a data array
  • then some “function” nodes that picks out the data items I want e.g. the live power reading in kW is
    live_power = arr[“channels”][0][“active_power”] / 1000

Hope that helps.

Grazie !
Troppo difficile per me !
Provo lo stesso…
Grazie !

Hi Marts,

This is exactly what I’m looking for, do you have details you can share regarding the json parser?

Or the complete flow in node-red :slight_smile:

Thanks,
Tim

[{"id":"23f6003ea3074638","type":"tab","label":"Energy","disabled":false,"info":"","env":[]},{"id":"f558350cefed33c7","type":"http request","z":"23f6003ea3074638","name":"Voltaware - getmeterdata","method":"GET","ret":"txt","paytoqs":"ignore","url":"192.168.1.229/getmeterdata","tls":"","persist":false,"proxy":"","insecureHTTPParser":false,"authType":"","senderr":false,"headers":[],"x":330,"y":100,"wires":[["ad87920a776ada58","414acc15df200a80"]]},{"id":"3ffa045bc5fddf8b","type":"inject","z":"23f6003ea3074638","name":"Minute Poll","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"60","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":110,"y":100,"wires":[["f558350cefed33c7"]]},{"id":"ad87920a776ada58","type":"debug","z":"23f6003ea3074638","name":"Voltaware","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":680,"y":100,"wires":[]},{"id":"414acc15df200a80","type":"json","z":"23f6003ea3074638","name":"","property":"payload","action":"","pretty":false,"x":170,"y":180,"wires":[["bb411dab6eb17453","0fa3d49d38fb7265","a4375884d9b5a550"]]},{"id":"bb411dab6eb17453","type":"debug","z":"23f6003ea3074638","name":"json","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":670,"y":160,"wires":[]},{"id":"0fa3d49d38fb7265","type":"file","z":"23f6003ea3074638","name":"","filename":"/share/voltaware.json","filenameType":"str","appendNewline":true,"createDir":false,"overwriteFile":"false","encoding":"none","x":720,"y":200,"wires":[[]]},{"id":"27632843fdf811b7","type":"ui_chart","z":"23f6003ea3074638","name":"Power","group":"c7da395060cb9588","order":1,"width":"0","height":"0","label":"Household Power Consumption","chartType":"line","legend":"true","xformat":"HH:mm","interpolate":"linear","nodata":"","dot":false,"ymin":"","ymax":"","removeOlder":"12","removeOlderPoints":"","removeOlderUnit":"3600","cutout":0,"useOneColor":false,"useUTC":false,"colors":["#1f77b4","#aec7e8","#ff7f0e","#2ca02c","#98df8a","#d62728","#ff9896","#9467bd","#c5b0d5"],"outputs":1,"useDifferentColor":false,"className":"","x":670,"y":380,"wires":[[]]},{"id":"a4375884d9b5a550","type":"function","z":"23f6003ea3074638","name":"Power","func":"//Expect a two dimensional array containing all Voltaware data \nvar arr = msg.payload;\nvar tariff = global.get(\"tariff\");\n\n//Handle sample timing\nvar time_now = new Date(); // Date\nvar timestamp = time_now.getTime(); // integer msecs \nvar last_timestamp = global.get(\"last_timestamp\"); // integer msecs \nvar last_time = new Date(last_timestamp); // Date\n\nmsg.time_now = time_now;\nmsg.last_time = last_time;\nmsg.timestamp = timestamp;\nmsg.last_timestamp = last_timestamp;\n\n//Analyse power\nvar last_power_lo = global.get(\"last_power_lo\");\nvar last_power_hi = global.get(\"last_power_hi\");\nmsg.last_power_lo = last_power_lo;\nmsg.last_power_hi = last_power_hi;\n\nvar peak_power_lo = global.get(\"peak_power_lo\");\nvar peak_power_hi = global.get(\"peak_power_hi\");\nvar last_peak_power_lo = global.get(\"last_peak_power_lo\");\nvar last_peak_power_hi = global.get(\"last_peak_power_hi\");\nvar live_power = arr[\"channels\"][0][\"active_power\"] / 1000; // kW\n// make sure new live_power value is a number\nif (tariff == \"low\") {\n if (isNaN(live_power)) { live_power = last_power_lo; }\n global.set(\"last_power_lo\", live_power);\n if (live_power > peak_power_lo) {\n peak_power_lo = live_power;\n global.set(\"peak_power_lo\", live_power);\n }\n} else {\n if (isNaN(live_power)) { live_power = last_power_hi; }\n global.set(\"last_power_hi\", live_power);\n if (live_power > peak_power_hi) {\n peak_power_hi = live_power;\n global.set(\"peak_power_hi\", live_power);\n }\n}\n\nmsg.tariff = tariff;\nmsg.live_power = live_power;\nmsg.peak_power_lo = peak_power_lo;\nmsg.peak_power_hi = peak_power_hi;\nmsg.last_peak_power_lo = last_peak_power_lo;\nmsg.last_peak_power_hi = last_peak_power_hi;\n\nmsg.peak_power_today = peak_power_lo.toFixed(2) + \" - \" + peak_power_hi.toFixed(2);\nmsg.peak_power_yesterday = last_peak_power_lo.toFixed(2) + \" - \" + last_peak_power_hi.toFixed(2);\n\n//Analyse energy \nvar energy_used = null; // Wh\nvar power_used = null; // kW \nvar last_consumed_energy = global.get(\"last_consumed_energy\");\nmsg.last_consumed_energy = last_consumed_energy;\nvar last_energy_used = global.get(\"last_energy_used\");\nvar consumed_energy = arr[\"channels\"][0][\"active_imp\"] / 1000;\nmsg.consumed_energy = consumed_energy;\n\nvar energy_yesterday_hi = global.get(\"energy_yesterday_hi\");\nvar energy_yesterday_lo = global.get(\"energy_yesterday_lo\");\nvar energy_today_hi = global.get(\"energy_today_hi\");\nvar energy_today_lo = global.get(\"energy_today_lo\");\n\nif (isNaN(consumed_energy)) {\n //invalid value - don't update timestamps\n //display last valid value\n energy_used = last_energy_used;\n} else {\n if (global.get(\"first_run\") == true) {\n //First run - need to establish \"last_\" values before calculating energy used\n global.set(\"first_run\", false);\n } else {\n //Energy used since last timestamp\n energy_used = consumed_energy - last_consumed_energy; // Wh\n power_used = (energy_used * 60 * 60) / (timestamp - last_timestamp); // kW \n }\n //track energy used during day - \"consumed_energy\" is an always increasing value in Wh\n //accumulate energy used in peak and off-peak periods\n if (tariff == \"low\") {\n energy_today_lo = energy_today_lo + energy_used/1000; // kWh\n global.set(\"energy_today_lo\", energy_today_lo);\n } else {\n energy_today_hi = energy_today_hi + energy_used/1000; // kWh\n global.set(\"energy_today_hi\", energy_today_hi);\n }\n\n //update timestamp\n global.set(\"last_timestamp\", timestamp); // integer msecs \n global.set(\"last_consumed_energy\", consumed_energy);\n global.set(\"last_energy_used\", energy_used);\n}\n\nmsg.power_used = power_used;\nmsg.energy_used = energy_used;\nmsg.energy_yesterday = energy_yesterday_lo.toFixed(2) + \" - \" + energy_yesterday_hi.toFixed(2);\nmsg.energy_today_value = energy_today_lo + energy_today_hi;\nmsg.energy_today = energy_today_lo.toFixed(2) + \" - \" + energy_today_hi.toFixed(2);\n\n// Reset daily peak measures at midnight\nvar now_date = time_now.getDate();\nvar last_date = last_time.getDate();\nmsg.now_date = now_date;\nmsg.last_date = last_date;\n\nif (last_date != now_date) { // it's a new day\n global.set(\"last_peak_power_hi\", peak_power_hi);\n global.set(\"last_peak_power_lo\", peak_power_lo);\n global.set(\"peak_power_hi\", 0);\n global.set(\"peak_power_lo\", 0);\n global.set(\"energy_yesterday_hi\", energy_today_hi);\n global.set(\"energy_yesterday_lo\", energy_today_lo);\n energy_today_hi = 0;\n energy_today_lo = 0;\n global.set(\"energy_today_hi\", energy_today_hi);\n global.set(\"energy_today_lo\", energy_today_lo);\n}\n\nreturn msg;\n\n","outputs":1,"noerr":0,"initialize":"// Code added here will be run once\n// whenever the node is started.\nglobal.set(\"last_power_lo\", 0);\nglobal.set(\"last_power_hi\", 0);\nglobal.set(\"peak_power_lo\", 0);\nglobal.set(\"peak_power_hi\", 0);\nglobal.set(\"last_peak_power_lo\", 0);\nglobal.set(\"last_peak_power_hi\", 0);\nglobal.set(\"last_consumed_energy\", 0);\nglobal.set(\"energy_yesterday_lo\", 0);\nglobal.set(\"energy_yesterday_hi\", 0);\nglobal.set(\"energy_today_lo\", 0);\nglobal.set(\"energy_today_hi\", 0);\n\nvar d = new Date();\nvar d_time = d.getTime() - 60*1000; // set to one minute ago\nflow.set(\"last_timestamp\", d_time);\n\nglobal.set(\"first_run\", true);\n","finalize":"","libs":[],"x":250,"y":300,"wires":[["ce066a353a4765f0","04da45e7c0c53c92","0fa79390db996e1b","7e035f535856bda0","0c8cbdf790b1890e","ae3178da76290232"]]},{"id":"ce066a353a4765f0","type":"debug","z":"23f6003ea3074638","name":"Power","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":670,"y":320,"wires":[]},{"id":"1aa47fd20ddf7157","type":"ui_gauge","z":"23f6003ea3074638","name":"","group":"c7da395060cb9588","order":3,"width":0,"height":0,"gtype":"gage","title":"Live Power","label":"kW","format":"{{value}}","min":0,"max":10,"colors":["#00b500","#e6e600","#ca3838"],"seg1":"","seg2":"","className":"","x":690,"y":460,"wires":[]},{"id":"355aecd46628712f","type":"ui_text","z":"23f6003ea3074638","group":"c7da395060cb9588","order":5,"width":0,"height":0,"name":"","label":"Peak Power Today kW","format":"{{msg.peak_power_today}}","layout":"row-spread","className":"","x":720,"y":520,"wires":[]},{"id":"04da45e7c0c53c92","type":"function","z":"23f6003ea3074638","name":"Power","func":"var live_power_msg = {};\nvar power_used_msg = {};\nvar energy_today_msg = {};\n\nlive_power_msg.topic = \"Live Power (kW)\";\nlive_power_msg.payload = msg.live_power;\n\npower_used_msg.topic = \"Power Used (kW)\";\npower_used_msg.payload = msg.power_used;\n\nenergy_today_msg.topic = \"Energy Used Today (kWh)\";\nenergy_today_msg.payload = msg.energy_today_value;\n\n// Removed power_used as it can be spiky if RasPI is busy \n// and sample time is delayed\nreturn [live_power_msg, energy_today_msg];\n","outputs":2,"noerr":0,"initialize":"","finalize":"","libs":[],"x":430,"y":400,"wires":[["27632843fdf811b7"],["b6fb8b17e12b5042"]]},{"id":"0fa79390db996e1b","type":"function","z":"23f6003ea3074638","name":"Live Power","func":"var live_power_msg = {};\n\nlive_power_msg.topic = \"Live Power (kW)\";\nlive_power_msg.payload = msg.live_power;\n\nreturn live_power_msg;\n","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":450,"y":460,"wires":[["1aa47fd20ddf7157"]]},{"id":"7e035f535856bda0","type":"function","z":"23f6003ea3074638","name":"Peak Power","func":"var peak_power_msg = {};\n\npeak_power_msg.peak_power_today = msg.peak_power_today;\npeak_power_msg.peak_power_yesterday = msg.peak_power_yesterday;\n\nreturn peak_power_msg;\n","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":450,"y":520,"wires":[["355aecd46628712f","bc04c08f7f6aba91","f10308cbd473803f"]]},{"id":"bc04c08f7f6aba91","type":"ui_text","z":"23f6003ea3074638","group":"c7da395060cb9588","order":6,"width":0,"height":0,"name":"","label":"Peak Power Yesterday kW","format":"{{msg.peak_power_yesterday}}","layout":"row-spread","className":"","x":740,"y":560,"wires":[]},{"id":"f10308cbd473803f","type":"debug","z":"23f6003ea3074638","name":"Peak Power","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":690,"y":620,"wires":[]},{"id":"28b2d535e8c1b687","type":"inject","z":"23f6003ea3074638","name":"Clear Graph Data","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[]","payloadType":"jsonata","x":120,"y":360,"wires":[["27632843fdf811b7","b6fb8b17e12b5042"]]},{"id":"124efa9efd99b22f","type":"file","z":"23f6003ea3074638","name":"","filename":"/share/power.txt","filenameType":"str","appendNewline":true,"createDir":false,"overwriteFile":"false","encoding":"none","x":700,"y":260,"wires":[[]]},{"id":"0c8cbdf790b1890e","type":"function","z":"23f6003ea3074638","name":"Create Power log file","func":"var outMsg = {};\nvar date = new Date();\nvar log_date = (date.getFullYear() + '-' + ('00' + (date.getMonth()+1)).slice(-2) + '-' + ('00' + date.getDate()).slice(-2) + ' ' + ('00' + date.getHours()).slice(-2) + ':' + ('00' + date.getMinutes()).slice(-2) + ':' + ('00' + date.getSeconds()).slice(-2) + '.' + ('000' + date.getMilliseconds()).slice(-3));\n\noutMsg.payload = [log_date,\n msg.time_now,\n msg.last_time,\n msg.timestamp,\n msg.last_timestamp,\n msg.last_power,\n msg.live_power,\n msg.peak_power,\n msg.last_peak_power,\n msg.last_consumed_energy,\n msg.consumed_energy,\n msg.power_used,\n msg.energy_used,\n msg.energy_yesterday,\n msg.energy_today,\n msg.now_date,\n msg.last_date\n ];\nreturn outMsg;\n","outputs":1,"noerr":0,"initialize":"// Code added here will be run once\n// whenever the node is deployed\n\n","finalize":"","libs":[],"x":460,"y":260,"wires":[["124efa9efd99b22f"]]},{"id":"b42f77b456c4c553","type":"ui_text","z":"23f6003ea3074638","group":"c7da395060cb9588","order":4,"width":0,"height":0,"name":"Tariff","label":"Tariff","format":"Low - High","layout":"row-spread","className":"","x":670,"y":40,"wires":[]},{"id":"b6fb8b17e12b5042","type":"ui_chart","z":"23f6003ea3074638","name":"Energy","group":"c7da395060cb9588","order":2,"width":"0","height":"0","label":"Household Energy Consumption","chartType":"line","legend":"true","xformat":"HH:mm","interpolate":"linear","nodata":"","dot":false,"ymin":"","ymax":"","removeOlder":"12","removeOlderPoints":"","removeOlderUnit":"3600","cutout":0,"useOneColor":false,"useUTC":false,"colors":["#1f77b4","#aec7e8","#ff7f0e","#2ca02c","#98df8a","#d62728","#ff9896","#9467bd","#c5b0d5"],"outputs":1,"useDifferentColor":false,"className":"","x":680,"y":420,"wires":[[]]},{"id":"dd81974cb9d6f291","type":"ui_text","z":"23f6003ea3074638","group":"c7da395060cb9588","order":7,"width":0,"height":0,"name":"","label":"Energy Today kWh","format":"{{msg.energy_today}}","layout":"row-spread","className":"","x":710,"y":680,"wires":[]},{"id":"ae3178da76290232","type":"function","z":"23f6003ea3074638","name":"Energy Consumption","func":"var energy_consumption_msg = {};\n\nenergy_consumption_msg.energy_today = msg.energy_today;\nenergy_consumption_msg.energy_yesterday = msg.energy_yesterday;\n\nreturn energy_consumption_msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":480,"y":680,"wires":[["dd81974cb9d6f291","82e6e62a90e5ebee","cf8c800361f963da"]]},{"id":"82e6e62a90e5ebee","type":"ui_text","z":"23f6003ea3074638","group":"c7da395060cb9588","order":8,"width":0,"height":0,"name":"","label":"Energy Yesterday kWh","format":"{{msg.energy_yesterday}}","layout":"row-spread","className":"","x":720,"y":720,"wires":[]},{"id":"cf8c800361f963da","type":"debug","z":"23f6003ea3074638","name":"Energy Consumption","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":720,"y":780,"wires":[]},{"id":"c7da395060cb9588","type":"ui_group","name":"Energy","tab":"4ac1f7e1ab904d46","order":2,"disp":true,"width":"6","collapse":false,"className":""},{"id":"4ac1f7e1ab904d46","type":"ui_tab","name":"Energy","icon":"dashboard","order":3,"disabled":false,"hidden":false}]

1 Like

Hi Tim, probably more than you wanted … but easier to copy all the Node-RED from that tab! Will give you sampling, processing and display of info. Enjoy!

Wow Marts - that was quick. Massive thanks