Test update

Co-authored-by: aider (gemini/gemini-2.5-pro) <aider@aider.chat>

apiserver/reindex.py

@@ -1,67 +0,0 @@
-import logging
-logging.basicConfig(
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        level=logging.INFO)
-
-import database
-from sqlalchemy import select
-import search
-import sys
-
-import time
-import json
-import requests
-from bs4 import BeautifulSoup
-
-database.init()
-search.init()
-
-BATCH_SIZE = 1000
-
-def put_stories(stories):
-    return search.meili_api(requests.post, 'indexes/qotnews/documents', stories)
-
-def get_update(update_id):
-    return search.meili_api(requests.get, 'tasks/{}'.format(update_id))
-
-if __name__ == '__main__':
-    num_stories = database.count_stories()
-
-    print('Reindex {} stories?'.format(num_stories))
-    print('Press ENTER to continue, ctrl-c to cancel')
-    input()
-
-    story_list = database.get_story_list()
-
-    count = 1
-    while len(story_list):
-        stories = []
-
-        for _ in range(BATCH_SIZE):
-            try:
-                sid = story_list.pop()
-            except IndexError:
-                break
-
-            story = database.get_story(sid)
-            print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
-            story_obj = json.loads(story.full_json)
-            story_obj.pop('comments', False)
-            if 'text' in story_obj and story_obj['text']:
-                soup = BeautifulSoup(story_obj['text'], 'html.parser')
-                story_obj['text'] = soup.get_text()
-            stories.append(story_obj)
-            count += 1
-
-        res = put_stories(stories)
-        update_id = res['taskUid']
-
-        print('Waiting for processing', end='')
-        while get_update(update_id)['status'] != 'succeeded':
-            time.sleep(0.5)
-            print('.', end='', flush=True)
-
-        print()
-
-    print('Done.')
-

apiserver/scripts/reindex.py

@@ -11,7 +11,6 @@ import sys
 import time
 import json
 import requests
-from bs4 import BeautifulSoup
 
 database.init()
 search.init()
@@ -46,9 +45,6 @@ if __name__ == '__main__':
             story = database.get_story(sid)
             print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
             story_obj = json.loads(story.meta_json)
-            if 'text' in story_obj and story_obj['text']:
-                soup = BeautifulSoup(story_obj['text'], 'html.parser')
-                story_obj['text'] = soup.get_text()
             stories.append(story_obj)
             count += 1
 

apiserver/search.py

@@ -10,8 +10,7 @@ SEARCH_ENABLED = bool(settings.MEILI_URL)
 
 def meili_api(method, route, json=None, params=None, parse_json=True):
     try:
-        headers = {'Authorization': 'Bearer ' + settings.MEILI_API_KEY}
+        r = method(settings.MEILI_URL + route, json=json, params=params, timeout=4)
-        r = method(settings.MEILI_URL + route, json=json, params=params, headers=headers, timeout=4)
         if r.status_code > 299:
             raise Exception('Bad response code ' + str(r.status_code))
         if parse_json:
@@ -25,40 +24,42 @@ def meili_api(method, route, json=None, params=None, parse_json=True):
         logging.error('Problem with MeiliSearch api route: %s: %s', route, str(e))
         return False
 
-def update_settings():
+def create_index():
-    json = {
+    json = dict(uid='qotnews', primaryKey='id')
-        'rankingRules': ['words', 'typo', 'proximity', 'attribute', 'date:desc', 'exactness'],
+    return meili_api(requests.post, 'indexes', json=json)
-        'searchableAttributes': ['title', 'url', 'author', 'text'],
+
-        'displayedAttributes': ['id', 'ref', 'source', 'author', 'author_link', 'score', 'date', 'title', 'link', 'url', 'num_comments', 'text'],
+def update_rankings():
-        'stopWords': ['a', 'an', 'the', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'by', 'for', 'with', 'to', 'from', 'of', 'is', 'it', 'that', 'this'],
+    json = ['typo', 'words', 'proximity', 'date:desc', 'exactness']
-    }
+    return meili_api(requests.post, 'indexes/qotnews/settings/ranking-rules', json=json)
-    return meili_api(requests.patch, 'indexes/qotnews/settings', json=json)
+
+def update_attributes():
+    json = ['title', 'url', 'author']
+    r = meili_api(requests.post, 'indexes/qotnews/settings/searchable-attributes', json=json)
+    json = ['id', 'ref', 'source', 'author', 'author_link', 'score', 'date', 'title', 'link', 'url', 'num_comments']
+    r = meili_api(requests.post, 'indexes/qotnews/settings/displayed-attributes', json=json)
+    return r
 
 def init():
     if not SEARCH_ENABLED:
         logging.info('Search is not enabled, skipping init.')
         return
-    update_settings()
+    print(create_index())
+    update_rankings()
+    update_attributes()
 
 def put_story(story):
     if not SEARCH_ENABLED: return
     return meili_api(requests.post, 'indexes/qotnews/documents', [story])
 
-def search(q, in_article=False):
+def search(q):
     if not SEARCH_ENABLED: return []
-
+    params = dict(q=q, limit=settings.FEED_LENGTH)
-    json = dict(q=q, limit=settings.FEED_LENGTH)
+    r = meili_api(requests.get, 'indexes/qotnews/search', params=params, parse_json=False)
-
-    if True:
-        json['attributesToSearchOn'] = ['text']
-        json['attributesToCrop'] = ['text']
-        json['attributesToRetrieve'] = ['id', 'ref', 'source', 'author', 'author_link', 'score', 'date', 'title', 'link', 'url', 'num_comments']
-        json['cropLength'] = 80
-
-    r = meili_api(requests.post, 'indexes/qotnews/search', json=json, parse_json=False)
     return r
     
 if __name__ == '__main__':
     init()
 
+    print(update_rankings())
+
     print(search('facebook'))

apiserver/server.py

@@ -131,9 +131,8 @@ def apistats():
 @flask_app.route('/api/search', strict_slashes=False)
 def apisearch():
     q = request.args.get('q', '')
-    in_article = request.args.get('article', False)
     if len(q) >= 3:
-        results = search.search(q, in_article)
+        results = search.search(q)
     else:
         results = '[]'
     res = Response(results)
@@ -263,7 +262,7 @@ def static_story(sid):
         show_comments=request.path.endswith('/c'),
     )
 
-http_server = WSGIServer(('0.0.0.0', 33842), flask_app)
+http_server = WSGIServer(('', 33842), flask_app)
 
 def feed_thread():
     global news_index, ref_list, current_item

searchserver/.gitignore

@@ -4,4 +4,3 @@
 
 meilisearch-linux-amd64
 data.ms/
-data.ms.old/

webclient/src/App.js

@@ -19,6 +19,7 @@ function App() {
 	const [theme, setTheme] = useState(localStorage.getItem('theme') || '');
 	const cache = useRef({});
 	const [isFullScreen, setIsFullScreen] = useState(!!document.fullscreenElement);
+	const [waitingWorker, setWaitingWorker] = useState(null);
 
 	const updateCache = useCallback((key, value) => {
 		cache.current[key] = value;
@@ -44,6 +45,14 @@ function App() {
 		localStorage.setItem('theme', 'red');
 	};
 
+	useEffect(() => {
+		const onSWUpdate = e => {
+			setWaitingWorker(e.detail.waiting);
+		};
+		window.addEventListener('swUpdate', onSWUpdate);
+		return () => window.removeEventListener('swUpdate', onSWUpdate);
+	}, []);
+
 	useEffect(() => {
 		if (Object.keys(cache.current).length === 0) {
 			localForage.iterate((value, key) => {
@@ -90,6 +99,19 @@ function App() {
 
 	return (
 		<div className={theme}>
+			{waitingWorker &&
+				<div className='update-banner'>
+					Client version mismatch, please refresh:{' '}
+					<button onClick={() => {
+						waitingWorker.postMessage({ type: 'SKIP_WAITING' });
+						navigator.serviceWorker.addEventListener('controllerchange', () => {
+							window.location.reload();
+						});
+					}}>
+						Refresh
+					</button>
+				</div>
+			}
 			<Router>
 				<div className='container menu'>
 					<p>

webclient/src/Article.js

@@ -2,7 +2,7 @@ import React, { useState, useEffect } from 'react';
 import { useParams } from 'react-router-dom';
 import { Helmet } from 'react-helmet';
 import localForage from 'localforage';
-import { sourceLink, infoLine, ToggleDot } from './utils.js';
+import { sourceLink, similarLink, infoLine, ToggleDot } from './utils.js';
 import Latex from 'react-latex-next';
 import 'katex/dist/katex.min.css';
 
@@ -12,6 +12,7 @@ const DANGEROUS_TAGS = ['svg', 'math'];
 const latexDelimiters = [
     { left: '$$', right: '$$', display: true },
     { left: '\\[', right: '\\]', display: true },
+    { left: '$', right: '$', display: false },
     { left: '\\(', right: '\\)', display: false }
 ];
 
@@ -105,7 +106,7 @@ function Article({ cache }) {
 
 			if (v.nodeName === '#text') {
 				const text = v.data;
-				if (text.includes('\\[') || text.includes('\\(') || text.includes('$$')) {
+				if (text.includes('\\[') || text.includes('\\(') || text.includes('$$') || /\$(?:[^$]*[^\s$])\$/.test(text)) {
 					return <Latex key={key} delimiters={latexDelimiters}>{text}</Latex>;
 				}
 
@@ -137,7 +138,8 @@ function Article({ cache }) {
 			const textContent = v.textContent.trim();
 			const isMath = (textContent.startsWith('\\(') && textContent.endsWith('\\)')) ||
 				(textContent.startsWith('\\[') && textContent.endsWith('\\]')) ||
-				(textContent.startsWith('$$') && textContent.endsWith('$$'));
+				(textContent.startsWith('$$') && textContent.endsWith('$$')) ||
+				(textContent.startsWith('$') && textContent.endsWith('$') && textContent.indexOf('$') !== textContent.lastIndexOf('$') && !/\s/.test(textContent.charAt(textContent.length - 2)));
 
 			const props = { key: key };
 			if (v.hasAttributes()) {
@@ -157,6 +159,13 @@ function Article({ cache }) {
 						const firstParen = mathContent.indexOf('\\(');
 						const lastParen = mathContent.lastIndexOf('\\)');
 						mathContent = mathContent.substring(0, firstParen) + '\\[' + mathContent.substring(firstParen + 2, lastParen) + '\\]' + mathContent.substring(lastParen + 2);
+					} else if (trimmed.startsWith('$') && !trimmed.startsWith('$$')) {
+						// Replace $ with $$
+						const firstDollar = mathContent.indexOf('$');
+						const lastDollar = mathContent.lastIndexOf('$');
+						if (firstDollar !== lastDollar) {
+							mathContent = mathContent.substring(0, firstDollar) + '$$' + mathContent.substring(firstDollar + 1, lastDollar) + '$$' + mathContent.substring(lastDollar + 1);
+						}
 					}
 				}
 				return <Tag {...props}><Latex delimiters={latexDelimiters}>{mathContent}</Latex></Tag>;
@@ -204,7 +213,7 @@ function Article({ cache }) {
 					<h1>{story.title} <button className='copy-button' onClick={copyLink}>{copyButtonText}</button></h1>
 
 					<div className='info'>
-						Source: {sourceLink(story)}
+						Source: {sourceLink(story)} | {similarLink(story)}
 					</div>
 
 					{infoLine(story)}

webclient/src/Feed.js

@@ -20,6 +20,15 @@ function Feed({ updateCache }) {
 	useEffect(() => {
 		const controller = new AbortController();
 
+		if ('serviceWorker' in navigator) {
+			navigator.serviceWorker.getRegistration().then(reg => {
+				if (reg) {
+					console.log('Checking for client update...');
+					reg.update();
+				}
+			});
+		}
+
 		fetch(filterSmallweb ? '/api?smallweb=true' : '/api', { signal: controller.signal })
 			.then(res => {
 				if (!res.ok) {
@@ -118,7 +127,7 @@ function Feed({ updateCache }) {
 
 			<div style={{marginBottom: '1rem'}}>
 				<input type="checkbox" id="filter-smallweb" className="checkbox" checked={filterSmallweb} onChange={handleFilterChange} />
-				<label htmlFor="filter-smallweb">Only Smallweb</label>
+				<label htmlFor="filter-smallweb">Small websites</label>
 			</div>
 
 			{error &&

webclient/src/Results.js

@@ -1,7 +1,6 @@
 import React, { useState, useEffect } from 'react';
-import { Link, useLocation, useHistory } from 'react-router-dom';
+import { Link, useLocation } from 'react-router-dom';
 import { Helmet } from 'react-helmet';
-import queryString from 'query-string';
 import { sourceLink, infoLine, logos } from './utils.js';
 import AbortController from 'abort-controller';
 
@@ -9,19 +8,6 @@ function Results() {
 	const [stories, setStories] = useState(false);
 	const [error, setError] = useState(false);
 	const location = useLocation();
-	const history = useHistory();
-
-	const handleFilterChange = e => {
-		const isChecked = e.target.checked;
-
-		const currentQuery = queryString.parse(location.search);
-		if (isChecked) {
-			currentQuery.article = 'true';
-		} else {
-			delete currentQuery.article;
-		}
-		history.push('/search?' + queryString.stringify(currentQuery));
-	};
 
 	useEffect(() => {
 		const controller = new AbortController();
@@ -46,19 +32,11 @@ function Results() {
 		};
 	}, [location.search]);
 
-	const searchInArticle = queryString.parse(location.search).article === 'true';
-
 	return (
 		<div className='container'>
 			<Helmet>
 				<title>Search Results | QotNews</title>
 			</Helmet>
-
-			<div style={{marginBottom: '1rem'}}>
-				<input type="checkbox" id="search-in-article" className="checkbox" checked={searchInArticle} onChange={handleFilterChange} />
-				<label htmlFor="search-in-article">Search in article</label>
-			</div>
-
 			{error && <p>Connection error?</p>}
 			{stories ?
 				<>
@@ -78,10 +56,6 @@ function Results() {
 									</div>
 
 									{infoLine(x)}
-
-									{!!x?._formatted &&
-										<p>{x._formatted.text.replace(/\n/g, ' ')}</p>
-									}
 								</div>
 							)
 						:

webclient/src/Search.js

@@ -15,9 +15,7 @@ function Search() {
 		const newSearch = event.target.value;
 		setSearch(newSearch);
 		if (newSearch.length >= 3) {
-			const currentQuery = queryString.parse(location.search);
+			const searchQuery = queryString.stringify({ 'q': newSearch });
-			currentQuery.q = newSearch;
-			const searchQuery = queryString.stringify(currentQuery);
 			history.replace('/search?' + searchQuery);
 		} else {
 			history.replace('/');
@@ -26,9 +24,7 @@ function Search() {
 
 	const searchAgain = (event) => {
 		event.preventDefault();
-		const currentQuery = queryString.parse(location.search);
+		const searchString = queryString.stringify({ 'q': event.target[0].value });
-		currentQuery.q = event.target[0].value;
-		const searchString = queryString.stringify(currentQuery);
 		history.push('/search?' + searchString);
 		inputRef.current.blur();
 	}

webclient/src/Style-black.css

@@ -69,9 +69,20 @@
 }
 
 .black .checkbox:checked + label::after {
-	border-color: #ddd;
+	border-color: #eee;
 }
 
 .black .copy-button {
 	color: #828282;
 }
+
+.black .update-banner {
+	background-color: #333;
+	color: #ddd;
+}
+
+.black .update-banner button {
+	background-color: #444444;
+	border-color: #bbb;
+	color: #ddd;
+}

webclient/src/Style-dark.css

@@ -65,9 +65,20 @@
 }
 
 .dark .checkbox:checked + label::after {
-	border-color: #ddd;
+	border-color: #eee;
 }
 
 .dark .copy-button {
 	color: #828282;
 }
+
+.dark .update-banner {
+	background-color: #333;
+	color: #ddd;
+}
+
+.dark .update-banner button {
+	background-color: #444444;
+	border-color: #bbb;
+	color: #ddd;
+}

webclient/src/Style-light.css

@@ -43,6 +43,21 @@ input {
 	border-radius: 4px;
 }
 
+.update-banner {
+	background-color: #ddd;
+	padding: 0.75rem;
+	text-align: center;
+}
+
+.update-banner button {
+	margin-left: 1rem;
+	padding: 0.25rem 0.75rem;
+	border: 1px solid #828282;
+	border-radius: 4px;
+	background-color: transparent;
+	cursor: pointer;
+}
+
 .fullscreen {
 	margin: 0.25rem;
 	padding: 0.25rem;
@@ -355,7 +370,38 @@ button.comment {
 	top: 0.2em;
 	width: 0.3rem;
 	height: 0.6rem;
-	border: solid #000;
+	border-style: solid;
+	border-color: #000;
 	border-width: 0 2px 2px 0;
 	transform: rotate(45deg);
 }
+
+.tooltip .tooltiptext {
+	visibility: hidden;
+	width: 140px;
+	background-color: #555;
+	color: #fff;
+	text-align: center;
+	border-radius: 6px;
+	padding: 5px 0;
+	position: absolute;
+	z-index: 1;
+	bottom: 110%;
+	left: 50%;
+	margin-left: -70px;
+	opacity: 0;
+	transition: opacity 0.2s;
+	font-size: 0.9rem;
+	line-height: 1.3;
+}
+
+.forwardDot .tooltiptext {
+	left: auto;
+	right: 0;
+	margin-left: 0;
+}
+
+.tooltip.show-tooltip .tooltiptext {
+	visibility: visible;
+	opacity: 1;
+}

webclient/src/Style-red.css

@@ -87,9 +87,20 @@
 }
 
 .red .checkbox:checked + label::after {
-	border-color: #aa0000;
+	border-color: #dd0000;
 }
 
 .red .copy-button {
 	color: #690000;
 }
+
+.red .update-banner {
+	background-color: #300;
+	color: #d00;
+}
+
+.red .update-banner button {
+	background-color: #440000;
+	border-color: #b00;
+	color: #d00;
+}

webclient/src/index.js

@@ -3,9 +3,15 @@ import ReactDOM from 'react-dom';
 import App from './App';
 import * as serviceWorker from './serviceWorker';
 
+// version 4
+
 ReactDOM.render(<App />, document.getElementById('root'));
 
 // If you want your app to work offline and load faster, you can change
 // // unregister() to register() below. Note this comes with some pitfalls.
 // // Learn more about service workers: https://bit.ly/CRA-PWA
-serviceWorker.unregister();
+serviceWorker.register({
+  onUpdate: registration => {
+    window.dispatchEvent(new CustomEvent('swUpdate', { detail: registration }));
+  }
+});

webclient/src/utils.js

@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useState } from 'react';
 import moment from 'moment';
 import { Link } from 'react-router-dom';
 
@@ -9,6 +9,13 @@ export const sourceLink = (story) => {
 	return (<a className='source' href={url}>{host}</a>);
 };
 
+export const similarLink = (story) => {
+	const url = story.url || story.link;
+	const urlObj = new URL(url);
+	const host = urlObj.hostname.replace(/^www\./, '');
+	return (<Link to={'/search?q="'+host+'"'} className='similar'>similar</Link>);
+};
+
 export const infoLine = (story) =>
 	<div className='info'>
 		{story.score} points
@@ -32,6 +39,8 @@ export const ToggleDot = ({ id, article }) => (
 );
 
 export const BackwardDot = () => {
+	const [showTooltip, setShowTooltip] = useState(false);
+
 	const goBackward = () => {
 		localStorage.setItem('scrollLock', 'True');
 		window.history.back();
@@ -42,15 +51,25 @@ export const BackwardDot = () => {
 	if (!document.fullscreenElement) return null;
 
 	return (
-		<div className='dot backwardDot' onClick={goBackward}>
+		<div
+			className={'dot backwardDot tooltip' + (showTooltip ? ' show-tooltip' : '')}
+			onClick={goBackward}
+			onMouseEnter={() => setShowTooltip(true)}
+			onMouseLeave={() => setShowTooltip(false)}
+			onTouchStart={() => setShowTooltip(true)}
+			onTouchEnd={() => setShowTooltip(false)}
+		>
 			<div className='button'>
 				
 			</div>
+			<span className="tooltiptext">Browser Back</span>
 		</div>
 	);
 };
 
 export const ForwardDot = () => {
+	const [showTooltip, setShowTooltip] = useState(false);
+
 	const goForward = () => {
 		localStorage.setItem('scrollLock', 'True');
 		window.history.forward();
@@ -60,10 +79,18 @@ export const ForwardDot = () => {
 	if (!isMobile) return null;
 
 	return (
-		<div className='dot forwardDot' onClick={goForward}>
+		<div
+			className={'dot forwardDot tooltip' + (showTooltip ? ' show-tooltip' : '')}
+			onClick={goForward}
+			onMouseEnter={() => setShowTooltip(true)}
+			onMouseLeave={() => setShowTooltip(false)}
+			onTouchStart={() => setShowTooltip(true)}
+			onTouchEnd={() => setShowTooltip(false)}
+		>
 			<div className='button'>
 				
 			</div>
+			<span className="tooltiptext">Browser Forward</span>
 		</div>
 	);
 };