Merge #6137

6137: Improve move_sqlite script to adapt timezone r=def- a=Zwelf

Fixes #6105

This change does some more stuff:

* change the filename to include the time with seconds to be able to run the script easier more often
* Fix that _backup tables are only moved after 1h and not immidiately in UTC+1
* Add a parameter to configure the number of minutes the backup
* Add parameter to change timestamps to be in localtime
* consider the timeout in num_transfer calculation
* remove rows in _backup tables to not print in them in the log
* rename tables to TABLES, because it is meant to be a constant

## Checklist

- [x] Tested the change ingame relative throughout
- [ ] Provided screenshots if it is a visual change
- [ ] Tested in combination with possibly related configuration options
- [ ] Written a unit test (especially base/) or added coverage to integration test
- [ ] Considered possible null pointers and out of bounds array indexing
- [ ] Changed no physics that affect existing maps
- [ ] Tested the change with [ASan+UBSan or valgrind's memcheck](https://github.com/ddnet/ddnet/#using-addresssanitizer--undefinedbehavioursanitizer-or-valgrinds-memcheck) (optional)


Co-authored-by: Zwelf <zwelf@strct.cc>

scripts/move_sqlite.py

@@ -18,22 +18,24 @@ from time import strftime
 import os
 from datetime import datetime, timedelta
 
-tables = ['record_race', 'record_teamrace', 'record_saves']
-date = (datetime.now() - timedelta(hours=1)).strftime('%Y-%m-%d %H:%M:%S')
+TABLES = ['record_race', 'record_teamrace', 'record_saves']
 
 def sqlite_table_exists(cursor, table):
 	cursor.execute(f"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='{table}'")
 	return cursor.fetchone()[0] != 0
 
-def sqlite_num_transfer(conn, table):
+def sqlite_num_transfer(conn, table, date):
 	c = conn.cursor()
 	if not sqlite_table_exists(c, table):
 		return 0
-	c.execute(f'SELECT COUNT(*) FROM {table}')
+	query = f'SELECT COUNT(*) FROM {table}'
+	if date is not None:
+		query += f' WHERE Timestamp < DATETIME("{date}", "utc")'
+	c.execute(query)
 	num = c.fetchone()[0]
 	return num
 
-def transfer(file_from, file_to):
+def transfer(file_from, file_to, date, keep_timestamp_utc):
 	conn_to = sqlite3.connect(file_to, isolation_level='EXCLUSIVE')
 	cursor_to = conn_to.cursor()
 
@@ -41,26 +43,49 @@ def transfer(file_from, file_to):
 	conn_from.text_factory = lambda b: b.decode(errors = 'ignore').rstrip()
 	for line in conn_from.iterdump():
 		cursor_to.execute(line)
-		print(line.encode('utf-8'))
-	for table in tables:
-		cursor_to.execute(f'INSERT INTO {table} SELECT * FROM {table}_backup WHERE Timestamp < "{date}"')
+	for table in TABLES:
+		cursor_to.execute(f'INSERT INTO {table} SELECT * FROM {table}_backup WHERE Timestamp < DATETIME("{date}", "utc")')
 	cursor_to.close()
 	conn_to.commit()
-	conn_to.close()
 
 	cursor_from = conn_from.cursor()
-	for table in tables:
+	for table in TABLES:
 		if sqlite_table_exists(cursor_from, table):
 			cursor_from.execute(f'DELETE FROM {table}')
 		backup_table = f'{table}_backup'
 		if sqlite_table_exists(cursor_from, backup_table):
-			cursor_from.execute(f'DELETE FROM {backup_table} WHERE Timestamp < "{date}"')
+			cursor_from.execute(f'DELETE FROM {backup_table} WHERE Timestamp < DATETIME("{date}", "utc")')
 	cursor_from.close()
 	conn_from.commit()
 	conn_from.close()
 
+	cursor_to = conn_to.cursor()
+	# delete non-moved backup-rows:
+	for table in TABLES:
+		backup_table = f'{table}_backup'
+		if sqlite_table_exists(cursor_to, backup_table):
+			cursor_to.execute(f'DELETE FROM {backup_table}')
+
+	if not keep_timestamp_utc:
+		# change date from utc to wanted current timezone for mysql https://github.com/ddnet/ddnet/issues/6105
+		for table in TABLES:
+			cursor_to.execute(f'''
+				UPDATE {table}
+				SET Timestamp = DATETIME(original.Timestamp, "localtime")
+				FROM (
+					SELECT rowid, Timestamp FROM {table}
+				) as original
+				WHERE {table}.rowid = original.rowid''')
+
+	cursor_to.close()
+	conn_to.commit()
+
+	for line in conn_to.iterdump():
+		print(line.encode('utf-8'))
+	conn_to.close()
+
 def main():
-	default_output = 'ddnet-server-' + strftime('%Y-%m-%d') + '.sqlite'
+	default_output = 'ddnet-server-' + strftime('%Y-%m-%dT%H:%M:%S') + '.sqlite'
 	parser = argparse.ArgumentParser(
 			description='Move DDNet ranks, teamranks and saves from a possible active SQLite3 to a new one',
 			formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -70,17 +95,27 @@ def main():
 	parser.add_argument('--to', '-t',
 			default=default_output,
 			help='Output file where ranks are saved adds current date by default')
+	parser.add_argument('--backup-timeout',
+			default=60,
+			type=int,
+			help='Time in minutes until when a rank is moved from the _backup tables')
+	parser.add_argument('--keep-timestamp-utc',
+			default=False,
+			action="store_true",
+			help='Timestamps are converted to localtime by default. To keep them utc set this config option')
 	args = parser.parse_args()
 
 	if not os.path.exists(args.f):
 		print(f"Warning: '{args.f}' does not exist (yet). Is the path specified correctly?")
 		return
 
+	date = (datetime.now() - timedelta(minutes=args.backup_timeout)).strftime('%Y-%m-%d %H:%M:%S')
+
 	conn = sqlite3.connect(args.f)
 	num = {}
-	for table in tables:
-		num[table] = sqlite_num_transfer(conn, table)
-		num[table] += sqlite_num_transfer(conn, f'{table}_backup')
+	for table in TABLES:
+		num[table] = sqlite_num_transfer(conn, table, None)
+		num[table] += sqlite_num_transfer(conn, f'{table}_backup', date)
 	conn.close()
 	if sum(num.values()) == 0:
 		return
@@ -95,7 +130,7 @@ def main():
 	print("Log of the transfer:")
 	print()
 
-	transfer(args.f, args.to)
+	transfer(args.f, args.to, date, args.keep_timestamp_utc)
 
 if __name__ == '__main__':
 	main()