diff --git a/Dockerfile b/Dockerfile index 71de5211..f310cec6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,10 +24,9 @@ RUN python app/setup_db.py # Expose port (Kinsta will set the PORT env var) EXPOSE 8080 -# Start the application. -# Run setup_db.py at startup (not just build time) so the persistent-volume -# database is migrated to the current schema before serving. setup_db.py is -# idempotent: it only adds missing columns/indexes on an existing DB. +# Start the application. Run setup_db.py first so the persistent DB volume is +# migrated on every deploy/restart (the build-time run above only touches the +# throwaway image layer, not the mounted data volume). setup_db is idempotent. CMD cd app && python setup_db.py && python serve.py --port ${PORT:-8080} --address 0.0.0.0 \ --allow-websocket-origin=ark-flight-review-9cuak.kinsta.app \ --host=ark-flight-review-9cuak.kinsta.app:443 diff --git a/app/cleanup_pending_logs.py b/app/cleanup_pending_logs.py new file mode 100644 index 00000000..3f6f3ff9 --- /dev/null +++ b/app/cleanup_pending_logs.py @@ -0,0 +1,113 @@ +#! /usr/bin/env python3 + +# Maintenance script: clean up logs left stuck in the Pending state. +# +# Uploads now require a registered & approved account and are parsed +# immediately, so no new log ever enters the Pending state. This script +# disposes of the historical backlog: +# +# - Anonymous pending logs (empty Uploader) predate the anonymous-upload ban. +# They are never parsed by policy; --delete removes their files and DB rows. +# - Pending logs with a named uploader are parsed automatically when that +# account is approved (process_pending_logs_for_user). They are listed here +# for visibility but never touched. +# +# Idempotent and list-only by default. Run from the app/ directory after the +# DB migration (setup_db.py): +# cd app && python cleanup_pending_logs.py # list only +# cd app && python cleanup_pending_logs.py --delete # remove anonymous logs + +import sys +import os +import argparse + +# this is needed for the following imports +sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'plot_app')) +from plot_app.config import get_db_connection, get_kml_filepath, \ + get_overview_img_filepath, get_log_filepath + +# Extensions a stored log file may have (see helper.get_log_filename) +LOG_FILE_EXTENSIONS = ('.ulg', '.bin', '.csv', '.bbl', '.txt') + + +def _pending_column_exists(cur) -> bool: + cur.execute("PRAGMA table_info('Logs')") + return any(row[1] == 'Pending' for row in cur.fetchall()) + + +def _unlink_if_exists(file_name): + if os.path.exists(file_name): + os.unlink(file_name) + return True + return False + + +def delete_log(cur, log_id): + """Remove a log's on-disk files and DB rows (mirrors EditEntryHandler).""" + kml_file_name = os.path.join(get_kml_filepath(), log_id.replace('/', '.') + '.kml') + _unlink_if_exists(kml_file_name) + preview_image = os.path.join(get_overview_img_filepath(), log_id + '.png') + _unlink_if_exists(preview_image) + for ext in LOG_FILE_EXTENSIONS: + _unlink_if_exists(os.path.join(get_log_filepath(), log_id + ext)) + cur.execute('DELETE FROM LogsGenerated WHERE Id = ?', (log_id,)) + cur.execute('DELETE FROM Logs WHERE Id = ?', (log_id,)) + + +def main(): + parser = argparse.ArgumentParser( + description='Clean up logs stuck in the Pending state. Lists by ' + 'default; --delete removes anonymous pending logs.') + parser.add_argument('--delete', action='store_true', + help='Delete anonymous pending logs (files and DB rows).') + args = parser.parse_args() + + con = get_db_connection() + try: + cur = con.cursor() + if not _pending_column_exists(cur): + print("The 'Pending' column does not exist yet — run setup_db.py first.") + return + cur.execute('select Id, Uploader, Date from Logs where Pending = 1') + rows = cur.fetchall() + + anonymous = [(log_id, date) for log_id, uploader, date in rows if not uploader] + named = [(log_id, uploader, date) for log_id, uploader, date in rows if uploader] + + print('Found {} pending log(s): {} anonymous, {} with a named uploader.' + .format(len(rows), len(anonymous), len(named))) + + for log_id, uploader, date in named: + print(' keeping {} (uploader={}, date={}) — parsed when the ' + 'account is approved'.format(log_id, uploader, date)) + + if not args.delete: + for log_id, date in anonymous: + print(' would delete {} (anonymous, date={})'.format(log_id, date)) + if anonymous: + print('Re-run with --delete to remove the anonymous log(s).') + return + + deleted = 0 + failed = 0 + for log_id, date in anonymous: + print('Deleting {} (anonymous, date={}) ... '.format(log_id, date), + end='', flush=True) + try: + delete_log(cur, log_id) + con.commit() + except Exception as e: # pylint: disable=broad-except + failed += 1 + print('error: {}'.format(e)) + continue + deleted += 1 + print('done') + + print('Deleted {}, failed {}, kept {} named.'.format( + deleted, failed, len(named))) + finally: + con.close() + + +if __name__ == '__main__': + main() diff --git a/app/tornado_handlers/upload.py b/app/tornado_handlers/upload.py index 9d2a8bbe..a63f3c08 100644 --- a/app/tornado_handlers/upload.py +++ b/app/tornado_handlers/upload.py @@ -13,6 +13,7 @@ import traceback import uuid import binascii +import tornado.escape import tornado.web from tornado.ioloop import IOLoop @@ -216,11 +217,23 @@ def initialize(self): def prepare(self): """ called before a new request """ if self.request.method.upper() == 'POST': + # Uploads require a registered & approved account. Reject here, + # before the request body is streamed, so anonymous clients don't + # transfer the whole file just to be turned away. The approval + # re-check covers accounts revoked after their login cookie was set. + username = self.current_user + if not username or not _is_uploader_approved(username): + _upload_log.warning('rejected_unauthenticated ip=%s uploader=%s', + client_ip(self), username or '-') + raise CustomHTTPError( + 403, 'Uploading requires a registered, approved account. ' + 'Please log in and try again.') + try: total = int(self.request.headers.get("Content-Length", "0")) except KeyError: total = 0 - + self.multipart_streamer = MultiPartStreamer(total) def data_received(self, chunk): @@ -238,6 +251,13 @@ def get(self, *args, **kwargs): self.redirect(url) return + # The upload form is only usable with an approved account (POSTs are + # rejected in prepare()), so send anonymous visitors to the login page. + # Viewing plots (the ?log= redirect above) stays public. + if not self.current_user: + self.redirect('/login?next=' + tornado.escape.url_escape('/upload')) + return + initial_email = '' # try to get the email from the cookie try: @@ -447,35 +467,28 @@ async def post(self, *args, **kwargs): # generate a token: secure random string (url-safe) token = str(binascii.hexlify(os.urandom(16)), 'ascii') - # Load the ulog file but only if not uploaded via CI. - # Also defer parsing entirely if the uploader is not an approved user - # (anonymous uploads or pending registrations) — the log will be - # parsed later when the user is approved. + # Load the ulog file but only if not uploaded via CI. Only + # approved accounts reach this point (enforced in prepare()), + # so parsing happens immediately — no deferred/pending state. ulog = None - is_pending = 0 if source != 'CI': - if uploader_username and _is_uploader_approved(uploader_username): - ulog_file_name = get_log_filename(log_id) - try: - ulog = await parse_log_bounded(ulog_file_name) - except UnsupportedLogFormat as e: - raise CustomHTTPError(400, str(e)) from e - except ParserTimeout as e: - _upload_log.warning( - 'parse_timeout ip=%s uploader=%s id=%s size=%s', - ip, uploader_username or '-', log_id, upload_size) - raise CustomHTTPError(400, - 'Log parsing took too long; the file may be corrupt or unsupported.') from e - except ParserCrashed as e: - _upload_log.error( - 'parse_crashed ip=%s uploader=%s id=%s size=%s', - ip, uploader_username or '-', log_id, upload_size) - raise CustomHTTPError(400, - 'Log parser failed unexpectedly on this file.') from e - else: - is_pending = 1 - print(f"Deferring parsing for log {log_id} " - f"(uploader='{uploader_username}' not approved)") + ulog_file_name = get_log_filename(log_id) + try: + ulog = await parse_log_bounded(ulog_file_name) + except UnsupportedLogFormat as e: + raise CustomHTTPError(400, str(e)) from e + except ParserTimeout as e: + _upload_log.warning( + 'parse_timeout ip=%s uploader=%s id=%s size=%s', + ip, uploader_username or '-', log_id, upload_size) + raise CustomHTTPError(400, + 'Log parsing took too long; the file may be corrupt or unsupported.') from e + except ParserCrashed as e: + _upload_log.error( + 'parse_crashed ip=%s uploader=%s id=%s size=%s', + ip, uploader_username or '-', log_id, upload_size) + raise CustomHTTPError(400, + 'Log parser failed unexpectedly on this file.') from e # put additional data into a DB con = get_db_connection() @@ -491,7 +504,7 @@ async def post(self, *args, **kwargs): datetime.datetime.now(), allow_for_analysis, obfuscated, source, stored_email, wind_speed, rating, feedback, upload_type, video_url, error_labels, is_public, - token, uploader_username, is_pending, content_hash]) + token, uploader_username, 0, content_hash]) if ulog is not None: vehicle_data = update_vehicle_db_entry(cur, ulog, log_id, vehicle_name) @@ -583,9 +596,9 @@ async def post(self, *args, **kwargs): send_admin_notification_email(admin_email, email, full_plot_url, delete_url, edit_url, info) _upload_log.info( - 'accepted ip=%s uploader=%s id=%s size=%s pending=%s source=%s type=%s hash=%s', + 'accepted ip=%s uploader=%s id=%s size=%s source=%s type=%s hash=%s', ip, uploader_username or '-', log_id, upload_size, - is_pending, source, upload_type, + source, upload_type, content_hash[:12] if content_hash else '-') if should_redirect: