Catch bad OwnTrack data, add WH51 soil sensor

Co-authored-by: aider (gemini/gemini-2.5-pro) <aider@aider.chat>

client/src/App.js

@@ -730,6 +730,7 @@ function Graphs({end, duration}) {
 			<Air name='Living Room Air' sensorName='Living Room' end={end} duration={duration} />
 			<Air name='Kitchen Air' sensorName='Kitchen' end={end} duration={duration} />
 			<Air name='Bedroom Air' sensorName='Bedroom' end={end} duration={duration} />
+			<Air name='Laundry Room Air' sensorName='Laundry Room' end={end} duration={duration} />
 			<Temperature name='Outside Temperature' sensorName='Outside' end={end} duration={duration} yDomain={[-40, 40]} showHumidity={false} showFreezingLine={true} />
 			<Temperature name='Bedroom Temperature' sensorName='Bedroom' end={end} duration={duration} yDomain={[15, 30]} showHumidity={true} showFreezingLine={false} />
 			<Temperature name='Nook Temperature' sensorName='Nook' end={end} duration={duration} yDomain={[15, 30]} showHumidity={true} showFreezingLine={false} />
@@ -743,6 +744,7 @@ function Graphs({end, duration}) {
 			<Lux name='Living Room Lux' sensorName='Living Room' end={end} duration={duration} />
 			<Lux name='Kitchen Lux' sensorName='Kitchen' end={end} duration={duration} />
 			<Lux name='Bedroom Lux' sensorName='Bedroom' end={end} duration={duration} />
+			<Lux name='Laundry Room Lux' sensorName='Laundry Room' end={end} duration={duration} />
 			<Soil name='Dumb Cane Soil Moisture' sensorName='Dumb Cane' end={end} duration={duration} />
 			<Soil name='Kitchen Pothos Soil Moisture' sensorName='Kitchen Pothos' end={end} duration={duration} />
 			<Soil name='Dracaena Soil Moisture' sensorName='Dracaena' end={end} duration={duration} />

main.py

@@ -23,6 +23,7 @@ import aiomqtt
 from datetime import datetime, timedelta
 import pytz
 TIMEZONE = pytz.timezone('America/Edmonton')
+import hashlib
 
 app = web.Application()
 http_session = None
@@ -104,6 +105,13 @@ class Sensor():
 
         return str(before) != str(after)
 
+    def check_cooldown(self):
+        if self.last_update and self.skip_cooldown and time.time() - self.last_update < self.skip_cooldown:
+            # ignore data point
+            return True
+        else:
+            return False
+
     def log(self):
         if not self.value:
             return
@@ -112,7 +120,7 @@ class Sensor():
             logging.debug('Skipping writing %s, data hasn\'t changed', self)
             return
 
-        if self.last_update and self.skip_cooldown and time.time() - self.last_update < self.skip_cooldown:
+        if self.check_cooldown():
             logging.debug('Skipping writing %s, cooldown limit', self)
             return
 
@@ -261,6 +269,32 @@ class SoilSensor(Sensor):
             except TypeError:
                 pass
 
+class WH51SoilSensor(Sensor):
+    type_ = 'soil'
+    bad_keys = [
+        'model',
+        'mic',
+        'battery_ok',
+    ]
+    update_period = 60*30
+
+class P2ProScaleSensor(Sensor):
+    type_ = 'scale'
+    skip_cooldown = 60.0 * 60 * 18  # 18 hours
+
+    def check_cooldown(self):
+        if 'weight' not in self.value:
+            return False
+
+        skip = super().check_cooldown()
+
+        if skip:
+            controller_message('Cooldown skipping scale weight: ' + str(self.value['weight']))
+
+        return skip
+
+
+
 class SolarSensor(Sensor):
     type_ = 'solar'
 
@@ -326,7 +360,7 @@ async def poll_sensors():
             await sensor.poll()
             sensor.check_update()
 
-        await asyncio.sleep(1)
+        await asyncio.sleep(5)
 
 async def process_data(data):
     sensor = sensors.get(data['id'])
@@ -374,19 +408,32 @@ async def fetch_mqtt():
 
 
 async def owntracks(request):
-    data = await request.json()
-    logging.debug('Web data: %s', str(data))
+    try:
+        data = await request.json()
+        logging.debug('Web data: %s', str(data))
 
-    if data.get('_type', '') == 'location':
-        data['id'] = data['topic'].split('/')[-1]
-        data['timestamp'] = datetime.utcfromtimestamp(data['tst'])
-        if 'inregions' in data:
-            data['inregions'] = ','.join(data['inregions'])
-        await process_data(data)
-    else:
-        logging.info('Not a location, skipping.')
+        if data.get('_type', '') == 'location':
+            data['id'] = data['topic'].split('/')[-1]
+            data['timestamp'] = datetime.utcfromtimestamp(data['tst'])
+            if 'inregions' in data:
+                data['inregions'] = ','.join(data['inregions'])
+            await process_data(data)
+        else:
+            logging.info('Not a location, skipping.')
 
-    return web.Response()
+        return web.json_response({'status': 'ok'}, status=200)
+    except BaseException as e:
+        logging.exception('Problem with OwnTracks data: {} - {}'.format(e.__class__.__name__, str(e)))
+        try:
+            logging.info('Data: %s', str(data))
+        except:
+            pass
+        return web.json_response({'error': str(e)}, status=500)
+
+
+def share_sha256(measurement, share_start, share_end, api_key):
+    s = f'{measurement}-{share_start}-{share_end}-{api_key}'
+    return hashlib.sha256(s.encode()).hexdigest()
 
 async def history(request):
     api_key = request.rel_url.query.get('api_key', '')
@@ -395,6 +442,14 @@ async def history(request):
     measurement = request.match_info.get('measurement')
     name = request.match_info.get('name')
 
+    share_start = request.rel_url.query.get('shareStart', '')
+    share_end = request.rel_url.query.get('shareEnd', '')
+    share_sig = request.rel_url.query.get('shareSig', '')
+
+    share_authed = share_sig == share_sha256(measurement, share_start, share_end, settings.SENSORS_API_KEY)
+    authed = authed or share_authed
+
+
     if not authed and measurement in ['owntracks', 'sleep']:
         return web.json_response([])
 
@@ -423,6 +478,9 @@ async def history(request):
     elif duration == 'month':
         start = end - timedelta(days=30)
         window = '1d'
+    elif duration == 'quarter':
+        start = end - timedelta(days=90)
+        window = '1d'
     elif duration == 'year':
         start = end - timedelta(days=365)
         window = '1d'
@@ -430,7 +488,7 @@ async def history(request):
         raise
 
     window = request.rel_url.query.get('window', window)
-    if window not in ['1m', '3m', '10m', '1h', '2h', '1d', '7d', '30d']:
+    if window not in ['1m', '3m', '10m', '30m', '1h', '2h', '1d', '7d', '30d']:
         raise
 
     if name == 'Water':
@@ -443,6 +501,13 @@ async def history(request):
     start = int(start.timestamp())
     end = int(end.timestamp())
 
+    if share_authed:
+        if start <= int(share_start):
+            start = int(share_start)
+        if end >= int(share_end):
+            end = int(share_end)
+
+
     if measurement == 'temperature':
         client = sensors_client
         q = 'select mean("temperature_C") as temperature_C, mean("humidity") as humidity from temperature where "name" = \'{}\' and time >= {}s and time < {}s group by time({}) fill(linear)'.format(name, start, end, window)
@@ -482,6 +547,99 @@ async def history(request):
 
     return web.json_response(result)
 
+
+async def search(request):
+    api_key = request.rel_url.query.get('api_key', '')
+    authed = api_key == settings.SENSORS_API_KEY
+
+    measurement = request.match_info.get('measurement')
+    name = request.match_info.get('name')
+
+    if not authed:
+        return web.json_response([])
+
+    if name not in [x.name for x in sensors.sensors]:
+        raise
+
+    if measurement != 'owntracks':
+        return web.json_response({'error': 'not implemented for this measurement'}, status=400)
+
+    try:
+        post_data = await request.json()
+    except json.JSONDecodeError:
+        return web.json_response({'error': 'invalid json'}, status=400)
+
+    params = request.rel_url.query
+    logging.info('Search request: meas=%s, name=%s, params=%s, data=%s',
+                 measurement, name, params, post_data)
+
+    areas = post_data.get('areas')
+    if not areas or not isinstance(areas, list):
+        return web.json_response({'error': 'invalid areas format'}, status=400)
+
+    try:
+        for area in areas:
+            _ = area['southWest']['lat']
+            _ = area['southWest']['lng']
+            _ = area['northEast']['lat']
+            _ = area['northEast']['lng']
+    except (KeyError, TypeError):
+        return web.json_response({'error': 'invalid area format in areas list'}, status=400)
+
+    client = sensors_client
+
+    where_clauses = []
+    for area in areas:
+        sw = area['southWest']
+        ne = area['northEast']
+        where_clauses.append(f'("lat" >= {sw["lat"]} and "lat" <= {ne["lat"]} and "lon" >= {sw["lng"]} and "lon" <= {ne["lng"]})')
+
+    full_where_clause = ' or '.join(where_clauses)
+
+    q = f'select "lat", "lon" from owntracks where "acc" < 100 and "name" = \'{name}\' and ({full_where_clause}) order by time asc'
+    points = list(client.query(q).get_points())
+
+    ranges = []
+    current_range = None
+    last_point_dt = None
+    # Use a 12-hour gap to distinguish between separate visits
+    GAP_THRESHOLD_HOURS = 12
+
+    for point in points:
+        point_time_str = point['time']
+        if '.' in point_time_str:
+            point_dt = datetime.strptime(point_time_str, '%Y-%m-%dT%H:%M:%S.%fZ')
+        else:
+            point_dt = datetime.strptime(point_time_str, '%Y-%m-%dT%H:%M:%SZ')
+
+        if current_range is None:
+            current_range = {'start': point_dt, 'end': point_dt}
+        else:
+            time_diff_hours = (point_dt - last_point_dt).total_seconds() / 3600
+            if time_diff_hours > GAP_THRESHOLD_HOURS:
+                ranges.append({
+                    'start': int(current_range['start'].timestamp()),
+                    'end': int(current_range['end'].timestamp())
+                })
+                current_range = {'start': point_dt, 'end': point_dt}
+            else:
+                current_range['end'] = point_dt
+
+        last_point_dt = point_dt
+
+    if current_range is not None:
+        ranges.append({
+            'start': int(current_range['start'].timestamp()),
+            'end': int(current_range['end'].timestamp())
+        })
+
+    return web.json_response(ranges)
+
+
+async def options_handler(request):
+    return web.Response()
+
+
 async def latest(request):
     result = dict()
     api_key = request.rel_url.query.get('api_key', '')
@@ -535,16 +693,22 @@ if __name__ == '__main__':
     app.router.add_get('/', index)
     app.router.add_post('/owntracks', owntracks)
     app.router.add_get('/history/{measurement}/{name}', history)
+    app.router.add_post('/search/{measurement}/{name}', search)
+    app.router.add_route('OPTIONS', '/search/{measurement}/{name}', options_handler)
     app.router.add_get('/latest', latest)
 
+    # serial, name
+    # API look up is done by name
+    # when retiring / reassigning a serial, change it to something impossible ie. 9999
     sensors.add(ThermostatSensor('thermostat2', '192.168.69.152', 'Venstar'))
     sensors.add(ERTSCMSensor('31005493', 'Water'))
     sensors.add(ERTSCMSensor('78628180', 'Gas'))
     sensors.add(OwnTracksSensor('owntracks1', 'OwnTracks'))
-    sensors.add(AirSensor('air1', 'Living Room'))
+    sensors.add(AirSensor('air9999', 'Living Room'))
+    sensors.add(AirSensor('air1', 'Laundry Room'))
     sensors.add(AirSensor('air2', 'Bedroom'))
     sensors.add(AirSensor('air3', 'Kitchen'))
-    sensors.add(Acurite606TX('185', 'Outside', 0.0))
+    sensors.add(Acurite606TX('252', 'Outside', 0.0))
     sensors.add(AcuRite6002RM('999999', 'Seeds', 0.0))     # A
     sensors.add(AcuRite6002RM('999998', 'Misc', 0.7, -1.0))     # A
     sensors.add(AcuRite6002RM('12516', 'Basement', 0.7, -1.0))     # A
@@ -553,6 +717,10 @@ if __name__ == '__main__':
     sensors.add(SleepSensor('sleep1', 'Bedroom'))
     sensors.add(SolarSensor('solar', 'Solar'))
     sensors.add(SoilSensor('soil1', 'Dumb Cane'))
+    sensors.add(SoilSensor('soil2', 'Kitchen Pothos'))
+    sensors.add(SoilSensor('soil3', 'Dracaena'))
+    sensors.add(WH51SoilSensor('0fafff', 'Side Garden'))
+    sensors.add(P2ProScaleSensor('scale1', 'Master Bathroom'))
 
     sensors.add(QotMotionSensor('qot_dc3c', 'Bedroom'))
     sensors.add(QotMotionSensor('qot_88c3', 'Lower Stairs Hi'))

mapper/package.json

@@ -8,6 +8,8 @@
     "@testing-library/user-event": "^12.1.10",
     "axios": "^0.21.1",
     "leaflet": "^1.9.4",
+    "leaflet-draw": "^1.0.4",
+    "leaflet-polylinedecorator": "^1.6.0",
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.34",
     "react": "^18.0.0",
@@ -15,6 +17,7 @@
     "react-dom": "^17.0.2",
     "react-is": "^17.0.2",
     "react-leaflet": "^4.2.1",
+    "react-leaflet-draw": "^0.20.6",
     "react-range-slider-input": "^3.0.7",
     "react-scripts": "4.0.3",
     "web-vitals": "^1.0.1"

mapper/src/App.css

@@ -60,6 +60,7 @@ h2 {
 
 .submenu h2 {
 	color: white;
+	font-size: 1.1em;
 }
 
 .submenu-header {
@@ -82,6 +83,7 @@ h2 {
 	color: white;
 	border-radius: 0;
 	border: 0;
+	font-family: sans-serif;
 }
 
 .menu button:hover {
@@ -92,10 +94,75 @@ h2 {
 	background-color: #666;
 }
 
+.menu button:disabled {
+	color: #ce7e7e;
+}
+
 .submenu button {
 	background-color: #666;
 }
 
+.submenu-checkbox-label {
+	/* Make it look like a button */
+	background-color: #666;
+	height: 2.5rem;
+	font-size: 1.5rem;
+	color: white;
+	cursor: pointer;
+	font-family: sans-serif;
+
+	/* Center content */
+	display: flex;
+	align-items: center;
+	justify-content: center;
+	gap: 0.5rem;
+}
+
+.submenu-checkbox-label:hover {
+	background-color: #999;
+}
+
+.submenu-checkbox-label input[type="checkbox"] {
+	/* Reset default styles */
+	appearance: none;
+	-webkit-appearance: none;
+	-moz-appearance: none;
+	background-color: transparent;
+	margin: 0;
+
+	/* Custom checkbox style */
+	font: inherit;
+	color: currentColor;
+	width: 0.75em;
+	height: 0.75em;
+	border: 0.1em solid currentColor;
+	border-radius: 0;
+	transform: translateY(-0.075em);
+
+	/* For the checkmark */
+	display: grid;
+	place-content: center;
+}
+
+/* Fix for Firefox Mobile rendering a black background on checked state,
+   especially when extensions like Dark Reader are active. */
+.submenu-checkbox-label input[type="checkbox"]:checked {
+	background-color: transparent !important;
+}
+
+.submenu-checkbox-label input[type="checkbox"]::before {
+	content: "";
+	width: 0.75em;
+	height: 0.75em;
+	transform: scale(0);
+	transition: 120ms transform ease-in-out;
+	background-color: white;
+}
+
+.submenu-checkbox-label input[type="checkbox"]:checked::before {
+	transform: scale(1);
+}
+
 .datepicker .rdtPicker {
 	background: none;
 	border: none;
@@ -104,8 +171,120 @@ h2 {
 	font-family: sans-serif;
 }
 
+.datepicker .rdtPicker .rdtDay.rdtDisabled {
+	color: #ce7e7e !important;
+}
+
+.datepicker .rdtPicker .rdtDay.rdtNew,
+.datepicker .rdtPicker .rdtDay.rdtOld {
+	color: #c8c8c8;
+}
+
 .datepicker th:hover,
 .datepicker td:hover {
 	background-color: #999!important;
 }
 
+.submenu-actions {
+	padding-top: 0.5rem;
+	display: flex;
+	flex-direction: column;
+	gap: 0.5rem;
+}
+
+.submenu-group {
+	display: flex;
+	align-items: center;
+	gap: 0.5rem;
+}
+
+.submenu-group span {
+	color: white;
+	flex-shrink: 0;
+}
+
+.submenu-group button {
+	flex-grow: 1;
+}
+
+.search-results-container {
+	max-height: 50vh;
+	overflow-y: auto;
+}
+
+.search-results-group-header {
+	color: white;
+	margin: 0.5em 0 0.25em;
+	font-size: 1em;
+	font-weight: normal;
+	text-align: center;
+}
+
+.search-results-empty {
+	color: white;
+	text-align: center;
+	padding: 1em 0;
+}
+
+.submenu button.active {
+	background-color: #4a4a4a;
+}
+
+.loading-overlay {
+	position: absolute;
+	top: 0;
+	left: 0;
+	width: 100%;
+	height: 100%;
+	background: rgba(0, 0, 0, 0.5);
+	z-index: 10000;
+	display: flex;
+	justify-content: center;
+	align-items: center;
+	color: white;
+}
+
+.search-result-row {
+    display: flex;
+    align-items: center;
+    position: relative;
+}
+
+.merge-button-wrapper {
+    flex-shrink: 0;
+    width: 2em;
+    position: relative;
+    align-self: stretch;
+}
+
+.merge-button {
+    position: absolute;
+    bottom: 0;
+    left: 50%;
+    transform: translate(-50%, 50%);
+    z-index: 10;
+    background-color: #888;
+    border: 1px solid #aaa;
+    border-radius: 50%;
+    color: white;
+    cursor: pointer;
+    font-size: 0.8rem;
+    line-height: 1;
+    height: 1.5em;
+    width: 1.5em;
+    padding: 0;
+    min-width: 0;
+}
+
+.merge-button:hover {
+    background-color: #999;
+}
+
+.search-result-button-wrapper {
+    flex-grow: 1;
+}
+
+.search-result-button-wrapper button {
+    width: 100%;
+    text-align: center;
+}

mapper/yarn.lock

@@ -7308,6 +7308,23 @@ last-call-webpack-plugin@^3.0.0:
     lodash "^4.17.5"
     webpack-sources "^1.1.0"
 
+leaflet-draw@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/leaflet-draw/-/leaflet-draw-1.0.4.tgz#45be92f378ed253e7202fdeda1fcc71885198d46"
+  integrity sha512-rsQ6saQO5ST5Aj6XRFylr5zvarWgzWnrg46zQ1MEOEIHsppdC/8hnN8qMoFvACsPvTioAuysya/TVtog15tyAQ==
+
+leaflet-polylinedecorator@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/leaflet-polylinedecorator/-/leaflet-polylinedecorator-1.6.0.tgz#9ef79fd1b5302d67b72efe959a8ecd2553f27266"
+  integrity sha512-kn3krmZRetgvN0wjhgYL8kvyLS0tUogAl0vtHuXQnwlYNjbl7aLQpkoFUo8UB8gVZoB0dhI4Tb55VdTJAcYzzQ==
+  dependencies:
+    leaflet-rotatedmarker "^0.2.0"
+
+leaflet-rotatedmarker@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/leaflet-rotatedmarker/-/leaflet-rotatedmarker-0.2.0.tgz#4467f49f98d1bfd56959bd9c6705203dd2601277"
+  integrity sha512-yc97gxLXwbZa+Gk9VCcqI0CkvIBC9oNTTjFsHqq4EQvANrvaboib4UdeQLyTnEqDpaXHCqzwwVIDHtvz2mUiDg==
+
 leaflet@^1.9.4:
   version "1.9.4"
   resolved "https://registry.yarnpkg.com/leaflet/-/leaflet-1.9.4.tgz#23fae724e282fa25745aff82ca4d394748db7d8d"
@@ -7386,6 +7403,11 @@ locate-path@^5.0.0:
   dependencies:
     p-locate "^4.1.0"
 
+lodash-es@^4.17.15:
+  version "4.17.21"
+  resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee"
+  integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==
+
 lodash._reinterpolate@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d"
@@ -9515,6 +9537,14 @@ react-is@^18.0.0:
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b"
   integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==
 
+react-leaflet-draw@^0.20.6:
+  version "0.20.6"
+  resolved "https://registry.yarnpkg.com/react-leaflet-draw/-/react-leaflet-draw-0.20.6.tgz#ba3ee41fea14d87ca610df9d248156367f2e921e"
+  integrity sha512-mGypDjJNrrnVpfKfGYovNBuJZXSk39ClOdUJe/5dB5Cj3f2BGQlY9txyV4UmUxZCbc96aq+FMwrGZeM4BokhHQ==
+  dependencies:
+    fast-deep-equal "^3.1.3"
+    lodash-es "^4.17.15"
+
 react-leaflet@^4.2.1:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/react-leaflet/-/react-leaflet-4.2.1.tgz#c300e9eccaf15cb40757552e181200aa10b94780"