Script to find items that no longer exist

Share your Projects! Scripts
1

Well, given the success of my previous script, it was inevitable that someone would ask for something else.

This is something I haven’t been using HA long enough to really need myself, but it will let you find items that are defined as part of a group, but don’t actually exist in the system any longer

I decided to use a weblink to display the name of the entity, since I couldn’t find just a basic text component, and a weblink is something I understand reasonably well, and assume it doesn’t add any functionality into the system. Once again, the group and elements it adds should be temporary, and not last across reboots.

So @Mariusthvdb and @arsaboo are at least partially responsible for this one :wink:

I’m not really happy with this right now, I can’t prevent the link from opening a new tab, and wish I didn’t have to use a link, and could just do plain text.

Anyway, here you go, and I will get the first comment so that I can keep the code up to date. Let me know what you think, and what enhancements this needs.

def process_group_entities(group, grouped_entities, hass, logger, process_group_entities):
#  logger.warn("processing group {}, currently {} grouped items".format(group.entity_id, len(grouped_entities)))

  for e in group.attributes["entity_id"]:
    domain = e.split(".")[0]
    if domain == "group":
      process_group_entities(hass.states.get(e), grouped_entities, hass, logger, process_group_entities)
    else:
      grouped_entities.add(e)
#  logger.warn("finishing group {}, currently {} grouped items".format(group.entity_id, len(grouped_entities)))

def scan_for_dead_entities(hass, logger, data, process_group_entities):
  target_group=data.get("target_group","deaditems")
  show_as_view = data.get("show_as_view", True)

  real_entities = set()
  grouped_entities = set()
  for s in hass.states.all():
    domain = s.entity_id.split(".")[0]
    if domain != "group":
      real_entities.add(s.entity_id)
    else:
      if (("view" not in s.attributes) or
          ( s.attributes["view"] == False)):
        real_entities.add(s.entity_id)
      process_group_entities(s, grouped_entities, hass, logger, process_group_entities)

  entity_ids=[]
  counter=0
  for e in (grouped_entities - real_entities):
    name = "weblink.deaditem{}".format(counter)
    hass.states.set(name, "javascript:return false", {"friendly_name":e})
    entity_ids.append(name)
    counter = counter +1

  service_data = {'object_id': target_group, 'name': 'Nonexisting Items',
                    'view': show_as_view, 'icon': 'mdi:cube-unfolded',
                    'control': 'hidden', 'entities': entity_ids,
                    'visible': True}

  hass.services.call('group', 'set', service_data, False)

scan_for_dead_entities(hass, logger, data, process_group_entities)
4 Likes
2

Script edited June 15th - prevent processing a single group multiple times, this should prevent cyclic loops and maximum recursion depth errors

Script edited June 16th - caught an issue where a group contains a non-existing group (i.e. in Group A, the entities contain group.doesntexist which was removed in a previous edit)

June 16th (second edit) - fixing recursion handling

def process_group_entities(group, grouped_entities, hass, logger, process_group_entities, processed_groups):
#  logger.warn("processing group {}, currently {} grouped items".format(group.entity_id, len(grouped_entities)))

  processed_groups.append(group.entity_id)
  for e in group.attributes["entity_id"]:
    domain = e.split(".")[0]
    if domain == "group":
      g = hass.states.get(e)
      if (g is not None) and (g.entity_id not in processed_groups):      
        process_group_entities(g, grouped_entities, hass, logger, process_group_entities, processed_groups)
#  logger.warn("finishing group {}, currently {} grouped items".format(group.entity_id, len(grouped_entities)))
  
def scan_for_dead_entities(hass, logger, data, process_group_entities):
  target_group=data.get("target_group","deaditems")
  show_as_view = data.get("show_as_view", True)

  real_entities = set()
  grouped_entities = set()
  processed_groups=[]
  
  for s in hass.states.all():
    domain = s.entity_id.split(".")[0]
    if domain != "group":
      real_entities.add(s.entity_id)
    else:
      if (("view" not in s.attributes) or
          ( s.attributes["view"] == False)):
        real_entities.add(s.entity_id)
      process_group_entities(s, grouped_entities, hass, logger, process_group_entities, processed_groups)
      
  entity_ids=[]
  counter=0
  for e in (grouped_entities - real_entities):
    name = "weblink.deaditem{}".format(counter)
    hass.states.set(name, "javascript:return false", {"friendly_name":e})
    entity_ids.append(name)
    counter = counter +1

  service_data = {'object_id': target_group, 'name': 'Nonexisting Items',
                    'view': show_as_view, 'icon': 'mdi:cube-unfolded',
                    'control': 'hidden', 'entities': entity_ids,
                    'visible': True}

  hass.services.call('group', 'set', service_data, False)

scan_for_dead_entities(hass, logger, data, process_group_entities)```
1 Like
3

Seeing this error

Error executing script: maximum recursion depth exceeded
Traceback (most recent call last):
  File "/srv/homeassistant/lib/python3.6/site-packages/homeassistant/components/python_script.py", line 166, in execute
    exec(compiled.code, restricted_globals, local)
  File "find_unused_entities.py", line 43, in <module>
  File "find_unused_entities.py", line 26, in scan_for_dead_entities
  File "find_unused_entities.py", line 7, in process_group_entities
  File "find_unused_entities.py", line 7, in process_group_entities
  File "find_unused_entities.py", line 7, in process_group_entities
  [Previous line repeated 980 more times]
  File "find_unused_entities.py", line 4, in process_group_entities
RecursionError: maximum recursion depth exceeded
4

Do you have a group which contains itself?

If so change line 6 from

if domain == "group":

To

if domain == "group" and group.entity_id != e.entity_id:
5

Nopes. My groups.yaml is here

6

Okay, I will have to look at this tomorrow after work, there must be a cycle somewhere in your groups.

7

this is what i get:

17
17857×364 63.9 KB

8

maybe this is of use, for showing the result of the scan:
I use it in another setting to report the status of many sensors on 1 card showing the sensor.summary created in this line of python. It uses the custom-ui Card Text only, to mitigate the 255 character limit of a card. Doesn’t show links though, so te result would be a mere list of items found.

##########################################################################################
# Summary update
# Custom card: /custom_ui/state-card-value_only.html
##########################################################################################

for group_desc in groups_desc:
    if group_desc != '' and not group_desc.endswith(': '):
        summary = '{}{}\n'.format(summary, group_desc)

# Add to final summary
summary = '{}\n{}\n{}\n{}'.format(summary, alarms_desc, activity_desc, mode_desc)

if show_card:
    hass.states.set('sensor.summary', '', {
        'custom_ui_state_card': 'state-card-value_only',
        'text': summary
    })

as you can see, many items are added in the final stage, but for this case, only the 1 item could be added, and kill the line

# Add to final summary
summary = '{}\n{}\n{}\n{}'.format(summary, alarms_desc, activity_desc, mode_desc)

make it a sensor.orphans and we’re set?

using the following coloring or styles ive set in the underlying card i have adapted accordingly, one could even emphasize the entities per domain (just thinking freely here…)

##########################################################################################
# Codes for text_colors declared in 
# Custom card: /custom_ui/state-card-value_only.html
##########################################################################################

#      case "*": return "bold";
#      case "/": return "italic";
#      case "!": return "red";
#      case "+": return "green";
#      case "=": return "yellow";
#      case "%": return "grey";
#      case "$": return "brown";
#      case "#": return "blue";
#      default:  return "normal";

groups_desc = ['!|-> Nobody home since ',
               '+- Hubs: all online',
               '+- Status: all online',
               '%|-> Nothing playing',
               '%|-> Nothing tracked',
               '!|-> No lights on since ',
               '!|-> No switches on since ',
               '!|-> No appliances on since ',
               '%|-> No Appliances active since ']

and

groups_format = ['+- Home: {} -- {}',
                 '!|-> Offline: {} -- {}',
                 '!|-> Status - Critical device offline: {} -- {}',
                 '#- Playing: {} -- {}',
                 '#- Tracked: {} -- {}',
                 '=- Lights on: {} -- {}',
                 '#- Switches on: {} -- {}',
                 '#- Appliances on: {} -- {}',
                 '+- Active: {} -- {}']
9

Hi guys, updated code is in the first response. I added some code to prevent processing a group multiple times, this should prevent the maximum recursion errors you were getting @arsaboo.

@mariusthvdb I will take a look at the customUI component, and probably post an update tomorrow.

I might add a parameter so that people don’t need to have the CustomUI pieces installed, but it would let you opt-in.

10

With the updated script, I am getting

Error executing script: 'NoneType' object is not subscriptable
Traceback (most recent call last):
  File "/srv/homeassistant/lib/python3.6/site-packages/homeassistant/components/python_script.py", line 166, in execute
    exec(compiled.code, restricted_globals, local)
  File "find_unused_entities.py", line 46, in <module>
  File "find_unused_entities.py", line 29, in scan_for_dead_entities
  File "find_unused_entities.py", line 7, in process_group_entities
  File "find_unused_entities.py", line 4, in process_group_entities
  File "/srv/homeassistant/lib/python3.6/site-packages/RestrictedPython/Eval.py", line 35, in default_guarded_getitem
    return ob[index]
TypeError: 'NoneType' object is not subscriptable
11

Thanks @arsaboo your groups.yaml file let me recreate the issue.

You and @mariusthvdb both (probably?) have a non-existing group contained within another group (ironically, this might be the group.catchall). That was causing an error.

The edit this morning should fix that for you. Let me know if you’re still getting errors.

Now to dig in to Custom UI

12

All right, now I am back to the previous error

Sat Jun 16 2018 09:47:48 GMT-0400 (Eastern Daylight Time)

Error executing script: maximum recursion depth exceeded
Traceback (most recent call last):
  File "/srv/homeassistant/lib/python3.6/site-packages/homeassistant/components/python_script.py", line 166, in execute
    exec(compiled.code, restricted_globals, local)
  File "find_unused_entities.py", line 47, in <module>
  File "find_unused_entities.py", line 30, in scan_for_dead_entities
  File "find_unused_entities.py", line 10, in process_group_entities
  File "find_unused_entities.py", line 10, in process_group_entities
  File "find_unused_entities.py", line 10, in process_group_entities
  [Previous line repeated 980 more times]
  File "find_unused_entities.py", line 4, in process_group_entities
RecursionError: maximum recursion depth exceeded
13

Okay, I think I fixed the recursion again.

I hate dealing with recursion issues, always feel like I’m spinning in circles :slight_smile:

14

ready to try, which is the latest version, have a link?

15

The code in the first reply is up to date

16

I’m not getting any errors, but it’s not finding a deliberately inserted non-existing entity.

group:
  Maintenance:
    control: hidden
    entities:
      - input_boolean.maintenance_mode
      - switch.debug
      - sensor.hline_1
      - script.quick_restart
      - script.quick_reboot
      - script.upgrade_hass
      - script.github_pull
      - sensor.hline_2
      - input_text.github_message
      - script.github_push
      - weblink.github
      - sensor.hline_3
      - script.sync_dropbox
      - weblink.dropbox
      - media_player.testing_testing_123

(last item doesn’t exist, surprisingly :smile: )

But deaditems group is empty…

Capture
Capture765×237 14.1 KB

(edit) I made a very minor change to the python script, just so that it doesn’t appear as it’s own view and just creates the group, which is then dumped in to my debug view, but I’m pretty confident that that hasn’t made any difference - I’ve uploaded it all to my github as it is now - everything (test entity, switch to activate the groups, and the group they land in) is in the Maintenance package…

link removed

My changes to the python_script were simply to remove the line show_as_view = data.get("show_as_view", True) from the scan_for_dead_entities function, and then the line that generates ‘service_data’ (line 38/39ish?) changed to

service_data = {'object_id': target_group, 'name': 'Nonexisting Items',
                    'view': False, 'control': 'hidden',
                    'entities': entity_ids, 'visible': True}

so it just generates as a normal group. The python script is in the usual place on Github too.

Thoughts?

17

no errors, an and no orphans… (changed dead_items in orphans, seemed better, friendly and more to the point…:wink: )

40
40875×680 49.6 KB

18

I’m having trouble figuring out how this works. I have my python_scripts folder and the script within it. I make a service call… then what? Where is the data that it generates?

EDIT: Figured it out.

19

having similar issues. group is not showing any ‘orphans’ even though i know i have them. group is also showing as unknown state as well as:

Group 'deaditems' doesn't exist!

using the package on mf_social’s Github (commented out most of it)

20

Hi all, oh boy I don’t know what happened yesterday, but that code didn’t work for me today either.

I have changed the name from deaditems to ghosts (they no longer exist, but they’ve left remnants all over the place).

Anyway, here’s the current version of the code, I also added support for the CustomUI component, although it is turned off by default. This uses Marius’ idea to add a sensor, and use the custom details to display text. I haven’t looked into colorizing it yet, but that is certainly on the roadmap.

def process_group_entities(group, grouped_entities, hass, logger, process_group_entities, processed_groups):
#  logger.warn("processing group {}, currently {} grouped items".format(group.entity_id, len(grouped_entities)))

  processed_groups.append(group.entity_id)
  for e in group.attributes["entity_id"]:
    domain = e.split(".")[0]
    if domain == "group":
      g = hass.states.get(e)
      if (g is not None) and (g.entity_id not in processed_groups):      
        process_group_entities(g, grouped_entities, hass, logger, process_group_entities, processed_groups)
    else:
      grouped_entities.add(e)
      
#  logger.warn("finishing group {}, currently {} grouped items".format(group.entity_id, len(grouped_entities)))
  
def scan_for_ghosts(hass, logger, data, process_group_entities):
  target_group=data.get("target_group","deaditems")
  show_as_view = data.get("show_as_view", True)
  use_custom_ui = data.get("use_custom_ui", False)

  real_entities = set()
  grouped_entities = set()
  processed_groups=[]
  
  for s in hass.states.all():
    domain = s.entity_id.split(".")[0]
    if domain != "group":
      real_entities.add(s.entity_id)
    else:
      if (("view" not in s.attributes) or
          ( s.attributes["view"] == False)):
        real_entities.add(s.entity_id)
      process_group_entities(s, grouped_entities, hass, logger, process_group_entities, processed_groups)
      
#  logger.error("{} real entities".format(len(real_entities)))
#  logger.error("{} grouped entities".format(len(grouped_entities)))
#  logger.error("{} groups processed".format(len(processed_groups)))
  results = grouped_entities - real_entities
#  logger.error("{} entities to list".format(len(results)))
  entity_ids=[]

  if use_custom_ui:
    summary=""
    for e in results:
      summary = "{}{}\n".format(summary, e)
    
    hass.states.set('sensor.ghost_items', '', {
        'custom_ui_state_card': 'state-card-value_only',
        'text': summary
    })
    entity_ids.append("sensor.ghost_items")
    
  else:
    counter=0
    for e in results:
      name = "weblink.deaditem{}".format(counter)
      hass.states.set(name, "javascript:return false", {"friendly_name":e})
      entity_ids.append(name)
      counter = counter +1

  service_data = {'object_id': target_group, 'name': 'Ghost Items',
                    'view': show_as_view, 'icon': 'mdi:cube-unfolded',
                    'control': 'hidden', 'entities': entity_ids,
                    'visible': True}

  hass.services.call('group', 'set', service_data, False)

scan_for_ghosts(hass, logger, data, process_group_entities)