feat(upload): require approved account for log uploads

Dockerfile

@@ -24,10 +24,9 @@ RUN python app/setup_db.py
 # Expose port (Kinsta will set the PORT env var)
 EXPOSE 8080
 
-# Start the application.
-# Run setup_db.py at startup (not just build time) so the persistent-volume
-# database is migrated to the current schema before serving. setup_db.py is
-# idempotent: it only adds missing columns/indexes on an existing DB.
+# Start the application. Run setup_db.py first so the persistent DB volume is
+# migrated on every deploy/restart (the build-time run above only touches the
+# throwaway image layer, not the mounted data volume). setup_db is idempotent.
 CMD cd app && python setup_db.py && python serve.py --port ${PORT:-8080} --address 0.0.0.0 \
     --allow-websocket-origin=ark-flight-review-9cuak.kinsta.app \
     --host=ark-flight-review-9cuak.kinsta.app:443

app/cleanup_pending_logs.py

@@ -0,0 +1,113 @@
+#! /usr/bin/env python3
+
+# Maintenance script: clean up logs left stuck in the Pending state.
+#
+# Uploads now require a registered & approved account and are parsed
+# immediately, so no new log ever enters the Pending state. This script
+# disposes of the historical backlog:
+#
+# - Anonymous pending logs (empty Uploader) predate the anonymous-upload ban.
+#   They are never parsed by policy; --delete removes their files and DB rows.
+# - Pending logs with a named uploader are parsed automatically when that
+#   account is approved (process_pending_logs_for_user). They are listed here
+#   for visibility but never touched.
+#
+# Idempotent and list-only by default. Run from the app/ directory after the
+# DB migration (setup_db.py):
+#     cd app && python cleanup_pending_logs.py           # list only
+#     cd app && python cleanup_pending_logs.py --delete  # remove anonymous logs
+
+import sys
+import os
+import argparse
+
+# this is needed for the following imports
+sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'plot_app'))
+from plot_app.config import get_db_connection, get_kml_filepath, \
+    get_overview_img_filepath, get_log_filepath
+
+# Extensions a stored log file may have (see helper.get_log_filename)
+LOG_FILE_EXTENSIONS = ('.ulg', '.bin', '.csv', '.bbl', '.txt')
+
+
+def _pending_column_exists(cur) -> bool:
+    cur.execute("PRAGMA table_info('Logs')")
+    return any(row[1] == 'Pending' for row in cur.fetchall())
+
+
+def _unlink_if_exists(file_name):
+    if os.path.exists(file_name):
+        os.unlink(file_name)
+        return True
+    return False
+
+
+def delete_log(cur, log_id):
+    """Remove a log's on-disk files and DB rows (mirrors EditEntryHandler)."""
+    kml_file_name = os.path.join(get_kml_filepath(), log_id.replace('/', '.') + '.kml')
+    _unlink_if_exists(kml_file_name)
+    preview_image = os.path.join(get_overview_img_filepath(), log_id + '.png')
+    _unlink_if_exists(preview_image)
+    for ext in LOG_FILE_EXTENSIONS:
+        _unlink_if_exists(os.path.join(get_log_filepath(), log_id + ext))
+    cur.execute('DELETE FROM LogsGenerated WHERE Id = ?', (log_id,))
+    cur.execute('DELETE FROM Logs WHERE Id = ?', (log_id,))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Clean up logs stuck in the Pending state. Lists by '
+                    'default; --delete removes anonymous pending logs.')
+    parser.add_argument('--delete', action='store_true',
+                        help='Delete anonymous pending logs (files and DB rows).')
+    args = parser.parse_args()
+
+    con = get_db_connection()
+    try:
+        cur = con.cursor()
+        if not _pending_column_exists(cur):
+            print("The 'Pending' column does not exist yet — run setup_db.py first.")
+            return
+        cur.execute('select Id, Uploader, Date from Logs where Pending = 1')
+        rows = cur.fetchall()
+
+        anonymous = [(log_id, date) for log_id, uploader, date in rows if not uploader]
+        named = [(log_id, uploader, date) for log_id, uploader, date in rows if uploader]
+
+        print('Found {} pending log(s): {} anonymous, {} with a named uploader.'
+              .format(len(rows), len(anonymous), len(named)))
+
+        for log_id, uploader, date in named:
+            print('  keeping {} (uploader={}, date={}) — parsed when the '
+                  'account is approved'.format(log_id, uploader, date))
+
+        if not args.delete:
+            for log_id, date in anonymous:
+                print('  would delete {} (anonymous, date={})'.format(log_id, date))
+            if anonymous:
+                print('Re-run with --delete to remove the anonymous log(s).')
+            return
+
+        deleted = 0
+        failed = 0
+        for log_id, date in anonymous:
+            print('Deleting {} (anonymous, date={}) ... '.format(log_id, date),
+                  end='', flush=True)
+            try:
+                delete_log(cur, log_id)
+                con.commit()
+            except Exception as e:  # pylint: disable=broad-except
+                failed += 1
+                print('error: {}'.format(e))
+                continue
+            deleted += 1
+            print('done')
+
+        print('Deleted {}, failed {}, kept {} named.'.format(
+            deleted, failed, len(named)))
+    finally:
+        con.close()
+
+
+if __name__ == '__main__':
+    main()

app/tornado_handlers/upload.py

@@ -13,6 +13,7 @@
 import traceback
 import uuid
 import binascii
+import tornado.escape
 import tornado.web
 from tornado.ioloop import IOLoop
 
@@ -216,11 +217,23 @@ def initialize(self):
     def prepare(self):
         """ called before a new request """
         if self.request.method.upper() == 'POST':
+            # Uploads require a registered & approved account. Reject here,
+            # before the request body is streamed, so anonymous clients don't
+            # transfer the whole file just to be turned away. The approval
+            # re-check covers accounts revoked after their login cookie was set.
+            username = self.current_user
+            if not username or not _is_uploader_approved(username):
+                _upload_log.warning('rejected_unauthenticated ip=%s uploader=%s',
+                                    client_ip(self), username or '-')
+                raise CustomHTTPError(
+                    403, 'Uploading requires a registered, approved account. '
+                         'Please log in and try again.')
+
             try:
                 total = int(self.request.headers.get("Content-Length", "0"))
             except KeyError:
                 total = 0
-            
+
             self.multipart_streamer = MultiPartStreamer(total)
 
     def data_received(self, chunk):
@@ -238,6 +251,13 @@ def get(self, *args, **kwargs):
             self.redirect(url)
             return
 
+        # The upload form is only usable with an approved account (POSTs are
+        # rejected in prepare()), so send anonymous visitors to the login page.
+        # Viewing plots (the ?log= redirect above) stays public.
+        if not self.current_user:
+            self.redirect('/login?next=' + tornado.escape.url_escape('/upload'))
+            return
+
         initial_email = ''
         # try to get the email from the cookie
         try:
@@ -447,35 +467,28 @@ async def post(self, *args, **kwargs):
                 # generate a token: secure random string (url-safe)
                 token = str(binascii.hexlify(os.urandom(16)), 'ascii')
 
-                # Load the ulog file but only if not uploaded via CI.
-                # Also defer parsing entirely if the uploader is not an approved user
-                # (anonymous uploads or pending registrations) — the log will be
-                # parsed later when the user is approved.
+                # Load the ulog file but only if not uploaded via CI. Only
+                # approved accounts reach this point (enforced in prepare()),
+                # so parsing happens immediately — no deferred/pending state.
                 ulog = None
-                is_pending = 0
                 if source != 'CI':
-                    if uploader_username and _is_uploader_approved(uploader_username):
-                        ulog_file_name = get_log_filename(log_id)
-                        try:
-                            ulog = await parse_log_bounded(ulog_file_name)
-                        except UnsupportedLogFormat as e:
-                            raise CustomHTTPError(400, str(e)) from e
-                        except ParserTimeout as e:
-                            _upload_log.warning(
-                                'parse_timeout ip=%s uploader=%s id=%s size=%s',
-                                ip, uploader_username or '-', log_id, upload_size)
-                            raise CustomHTTPError(400,
-                                'Log parsing took too long; the file may be corrupt or unsupported.') from e
-                        except ParserCrashed as e:
-                            _upload_log.error(
-                                'parse_crashed ip=%s uploader=%s id=%s size=%s',
-                                ip, uploader_username or '-', log_id, upload_size)
-                            raise CustomHTTPError(400,
-                                'Log parser failed unexpectedly on this file.') from e
-                    else:
-                        is_pending = 1
-                        print(f"Deferring parsing for log {log_id} "
-                              f"(uploader='{uploader_username}' not approved)")
+                    ulog_file_name = get_log_filename(log_id)
+                    try:
+                        ulog = await parse_log_bounded(ulog_file_name)
+                    except UnsupportedLogFormat as e:
+                        raise CustomHTTPError(400, str(e)) from e
+                    except ParserTimeout as e:
+                        _upload_log.warning(
+                            'parse_timeout ip=%s uploader=%s id=%s size=%s',
+                            ip, uploader_username or '-', log_id, upload_size)
+                        raise CustomHTTPError(400,
+                            'Log parsing took too long; the file may be corrupt or unsupported.') from e
+                    except ParserCrashed as e:
+                        _upload_log.error(
+                            'parse_crashed ip=%s uploader=%s id=%s size=%s',
+                            ip, uploader_username or '-', log_id, upload_size)
+                        raise CustomHTTPError(400,
+                            'Log parser failed unexpectedly on this file.') from e
 
                 # put additional data into a DB
                 con = get_db_connection()
@@ -491,7 +504,7 @@ async def post(self, *args, **kwargs):
                          datetime.datetime.now(), allow_for_analysis,
                          obfuscated, source, stored_email, wind_speed, rating,
                          feedback, upload_type, video_url, error_labels, is_public,
-                         token, uploader_username, is_pending, content_hash])
+                         token, uploader_username, 0, content_hash])
 
                     if ulog is not None:
                         vehicle_data = update_vehicle_db_entry(cur, ulog, log_id, vehicle_name)
@@ -583,9 +596,9 @@ async def post(self, *args, **kwargs):
                     send_admin_notification_email(admin_email, email, full_plot_url, delete_url, edit_url, info)
 
                 _upload_log.info(
-                    'accepted ip=%s uploader=%s id=%s size=%s pending=%s source=%s type=%s hash=%s',
+                    'accepted ip=%s uploader=%s id=%s size=%s source=%s type=%s hash=%s',
                     ip, uploader_username or '-', log_id, upload_size,
-                    is_pending, source, upload_type,
+                    source, upload_type,
                     content_hash[:12] if content_hash else '-')
 
                 if should_redirect: