You need to limit it to just entities with a defined device_class
:
{{ states.binary_sensor
| selectattr('attributes.device_class', 'defined')
| selectattr('attributes.device_class', 'eq', 'window')
| selectattr('state', 'search', 'off')
| rejectattr('object_id', 'search', 'battery')
| list | count }}