Fix search to work with low-RAM server

apiserver/reindex.py

@@ -8,12 +8,21 @@ from sqlalchemy import select
 import search
 import sys
 
+import time
 import json
 import requests
 
 database.init()
 search.init()
 
+BATCH_SIZE = 5000
+
+def put_stories(stories):
+    return search.meili_api(requests.post, 'indexes/qotnews/documents', stories)
+
+def get_update(update_id):
+    return search.meili_api(requests.get, 'indexes/qotnews/updates/{}'.format(update_id))
+
 def count_stories():
     try:
         session = database.Session()
@@ -35,11 +44,34 @@ if __name__ == '__main__':
     print('Press ENTER to continue, ctrl-c to cancel')
     input()
 
-    count = 1
-    for sid in get_story_list():
-        story = database.get_story(sid)
-        print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
-        story_obj = json.loads(story.meta_json)
-        search.put_story(story_obj)
-        count += 1
+    story_list = get_story_list()
+
+    count = 1
+    while len(story_list):
+        stories = []
+
+        for _ in range(BATCH_SIZE):
+            try:
+                sid = story_list.pop()
+            except IndexError:
+                break
+
+            story = database.get_story(sid)
+            print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
+            story_obj = json.loads(story.meta_json)
+            to_add = dict(title=story_obj['title'], id=story_obj['id'], date=story_obj['date'])
+            stories.append(to_add)
+            count += 1
+
+        res = put_stories(stories)
+        update_id = res['updateId']
+
+        print('Waiting for processing', end='')
+        while get_update(update_id)['status'] != 'processed':
+            time.sleep(0.5)
+            print('.', end='', flush=True)
+
+        print()
+
+    print('Done.')