Rewind ZSTD support

Common/Serialize/Serializer.h

@@ -255,7 +255,6 @@ class CChunkFileReader
 		}
 	}
 
-
 	// Load file template
 	template<class T>
 	static Error Load(const Path &filename, std::string *gitVersion, T& _class, std::string *failureReason)

Core/SaveState.cpp

@@ -20,6 +20,8 @@
 #include <thread>
 #include <mutex>
 
+#include <zstd.h>
+
 #include "Common/Data/Text/I18n.h"
 #include "Common/Thread/ThreadUtil.h"
 #include "Common/Data/Text/Parsers.h"
@@ -99,6 +101,17 @@ double g_lastSaveTime = -1.0;
 		return CChunkFileReader::LoadPtr(&data[0], state, errorString);
 	}
 
+	struct StateBuffer {
+		void Clear() {
+			zstd_compressed.clear();
+			decompressed_size = 0;
+			compressed_size = 0;
+		}
+		std::vector<u8> zstd_compressed;
+		size_t decompressed_size = 0;
+		size_t compressed_size = 0;
+	};
+
 	// This ring buffer of states is for rewind save states, which are kept in RAM.
 	// Save states are compressed against one of two reference saves (bases_), and the reference
 	// is switched to a fresh save every N saves, where N is BASE_USAGE_INTERVAL.
@@ -150,7 +163,7 @@ double g_lastSaveTime = -1.0;
 			if (err == CChunkFileReader::ERROR_NONE)
 				ScheduleCompress(&states_[n], compressBuffer, &bases_[base_]);
 			else
-				states_[n].clear();
+				states_[n].zstd_compressed.clear();
 
 			baseMapping_[n] = base_;
 			return err;
@@ -165,81 +178,135 @@ double g_lastSaveTime = -1.0;
 				return CChunkFileReader::ERROR_BAD_FILE;
 
 			int n = (--next_ + size_) % size_;
-			if (states_[n].empty())
+			if (states_[n].zstd_compressed.empty())
 				return CChunkFileReader::ERROR_BAD_FILE;
 
 			static std::vector<u8> buffer;
-			LockedDecompress(buffer, states_[n], bases_[baseMapping_[n]]);
-			CChunkFileReader::Error error = LoadFromRam(buffer, errorString);
-			rewindLastTime_ = time_now_d();
-			return error;
+			if (LockedDecompress(buffer, states_[n], bases_[baseMapping_[n]])) {
+				CChunkFileReader::Error error = LoadFromRam(buffer, errorString);
+				if (error == CChunkFileReader::ERROR_NONE) {
+					INFO_LOG(Log::SaveState, "Rewinding to recent savestate snapshot (%d bytes compressed)", states_[n].zstd_compressed.size());
+					rewindLastTime_ = time_now_d();
+				}
+				return error;
+			} else {
+				WARN_LOG(Log::SaveState, "Failed to load rewind savestate");
+				// Unclear what CChunkFileReader error code we should pass in this case, which I'm not sure will
+				// happen in practice barring memory corruption.
+			}
+			return CChunkFileReader::ERROR_NONE;
 		}
 
-		void ScheduleCompress(std::vector<u8> *result, const std::vector<u8> *state, const std::vector<u8> *base)
+		void ScheduleCompress(StateBuffer *result, const std::vector<u8> *state, const std::vector<u8> *base)
 		{
 			if (compressThread_.joinable())
 				compressThread_.join();
 			compressThread_ = std::thread([=]{
 				SetCurrentThreadName("SaveStateCompress");
 
 				// Should do no I/O, so no JNI thread context needed.
-				Compress(*result, *state, *base);
+				Compress(result, *state, *base);
 			});
 		}
 
-		void Compress(std::vector<u8> &result, const std::vector<u8> &state, const std::vector<u8> &base)
+		const bool USE_XOR = false;
+
+		void Compress(StateBuffer *result, const std::vector<u8> &state, const std::vector<u8> &base)
 		{
 			std::lock_guard<std::mutex> guard(lock_);
 			// Bail if we were cleared before locking.
 			if (first_ == 0 && next_ == 0)
 				return;
 
 			double start_time = time_now_d();
-			result.clear();
-			result.reserve(512 * 1024);
-			for (size_t i = 0; i < state.size(); i += BLOCK_SIZE)
-			{
-				int blockSize = std::min(BLOCK_SIZE, (int)(state.size() - i));
-				if (i + blockSize > base.size() || memcmp(&state[i], &base[i], blockSize) != 0)
+			std::vector<u8> compressed;
+			if (USE_XOR) {
+				compressed.resize(state.size());
+				for (size_t i = 0; i < state.size(); i++) {
+					if (i >= base.size()) {
+						compressed[i] = state[i];
+					} else {
+						compressed[i] = base[i] ^ state[i];
+					}
+				}
+			} else {
+				compressed.reserve(512 * 1024);
+				for (size_t i = 0; i < state.size(); i += BLOCK_SIZE)
 				{
-					result.push_back(1);
-					result.insert(result.end(), state.begin() + i, state.begin() + i + blockSize);
+					int blockSize = std::min(BLOCK_SIZE, (int)(state.size() - i));
+					if (i + blockSize > base.size() || memcmp(&state[i], &base[i], blockSize) != 0)
+					{
+						compressed.push_back(1);
+						compressed.insert(compressed.end(), state.begin() + i, state.begin() + i + blockSize);
+					} else {
+						compressed.push_back(0);
+					}
 				}
-				else
-					result.push_back(0);
 			}
 
 			double taken_s = time_now_d() - start_time;
-			DEBUG_LOG(Log::SaveState, "Rewind: Compressed save from %d bytes to %d in %0.2f ms.", (int)state.size(), (int)result.size(), taken_s * 1000.0);
+			DEBUG_LOG(Log::SaveState, "Rewind: Compressed save from %d bytes to %d in %0.2f ms.", (int)state.size(), (int)compressed.size(), taken_s * 1000.0);
+
+			// Temporarily allocate a buffer to do compression in.
+			size_t compressCapacity = ZSTD_compressBound(compressed.size());
+			result->zstd_compressed.resize(compressCapacity);
+			result->compressed_size = ZSTD_compress(&result->zstd_compressed[0], compressCapacity, compressed.data(), compressed.size(), 1);
+			if (result->compressed_size) {
+				result->zstd_compressed.resize(result->compressed_size);
+				result->decompressed_size = compressed.size();
+			}
+
+			double zstd_s = time_now_d() - start_time - taken_s;
+			DEBUG_LOG(Log::SaveState, "Rewind: ZSTD compressed to %d in %0.2f ms.", (int)result->compressed_size, zstd_s * 1000.0);
 		}
 
-		void LockedDecompress(std::vector<u8> &result, const std::vector<u8> &compressed, const std::vector<u8> &base)
+		bool LockedDecompress(std::vector<u8> &result, const StateBuffer &buffer, const std::vector<u8> &base)
 		{
 			result.clear();
 			result.reserve(base.size());
 			auto basePos = base.begin();
-			for (size_t i = 0; i < compressed.size(); )
-			{
-				if (compressed[i] == 0)
-				{
-					++i;
-					int blockSize = std::min(BLOCK_SIZE, (int)(base.size() - result.size()));
-					result.insert(result.end(), basePos, basePos + blockSize);
-					basePos += blockSize;
+
+			// OK, zstd decompress first.
+			std::vector<u8> compressed = std::vector<u8>(buffer.decompressed_size, 0);
+			size_t retval = ZSTD_decompress(&compressed[0], compressed.size(), buffer.zstd_compressed.data(), buffer.zstd_compressed.size());
+			if (ZSTD_isError(retval)) {
+				WARN_LOG(Log::SaveState, "Failed to decompress zstd-compressed rewind savestate");
+				return false;
+			}
+
+			if (USE_XOR) {
+				result.resize(compressed.size());
+				for (size_t i = 0; i < compressed.size(); i++) {
+					if (i < base.size()) {
+						result[i] = compressed[i] ^ base[i];
+					} else {
+						result[i] = compressed[i];
+					}
 				}
-				else
-				{
-					++i;
-					int blockSize = std::min(BLOCK_SIZE, (int)(compressed.size() - i));
-					result.insert(result.end(), compressed.begin() + i, compressed.begin() + i + blockSize);
-					i += blockSize;
-					// This check is to avoid advancing basePos out of range, which MSVC catches.
-					// When this happens, we're at the end of decoding anyway.
-					if (base.end() - basePos >= blockSize) {
+			} else {
+				for (size_t i = 0; i < compressed.size(); ) {
+					if (compressed[i] == 0) {
+						++i;
+						int blockSize = std::min(BLOCK_SIZE, (int)(base.size() - result.size()));
+						_dbg_assert_(blockSize >= 0);
+						result.insert(result.end(), basePos, basePos + blockSize);
 						basePos += blockSize;
+					} else {
+						++i;
+						int blockSize = std::min(BLOCK_SIZE, (int)(compressed.size() - i));
+						if (blockSize > 0) {
+							result.insert(result.end(), compressed.begin() + i, compressed.begin() + i + blockSize);
+							i += blockSize;
+							// This check is to avoid advancing basePos out of range, which MSVC catches.
+							// When this happens, we're at the end of decoding anyway.
+							if (base.end() - basePos >= blockSize) {
+								basePos += blockSize;
+							}
+						}
 					}
 				}
 			}
+			return true;
 		}
 
 		void Clear()
@@ -257,7 +324,7 @@ double g_lastSaveTime = -1.0;
 			baseMapping_.clear();
 			baseMapping_.resize(size_);
 			for (auto &s : states_) {
-				s.clear();
+				s.Clear();
 			}
 			buffer_.clear();
 			base_ = -1;
@@ -299,14 +366,12 @@ double g_lastSaveTime = -1.0;
 		// TODO: Instead, based on size of compressed state?
 		const int BASE_USAGE_INTERVAL = 15;
 
-		typedef std::vector<u8> StateBuffer;
-
 		int first_ = 0;
 		int next_ = 0;
 		int size_;
 
 		std::vector<StateBuffer> states_;
-		StateBuffer bases_[2];
+		std::vector<u8> bases_[2];
 		std::vector<int> baseMapping_;
 		std::mutex lock_;
 		std::thread compressThread_;
@@ -315,7 +380,7 @@ double g_lastSaveTime = -1.0;
 		int base_ = -1;
 		int baseUsage_ = 0;
 
-		double rewindLastTime_ = 0.0f;
+		double rewindLastTime_ = 0.0;
 	};
 
 	static bool needsProcess = false;
@@ -1067,7 +1132,6 @@ double g_lastSaveTime = -1.0;
 			}
 
 			case SAVESTATE_REWIND:
-				INFO_LOG(Log::SaveState, "Rewinding to recent savestate snapshot");
 				result = rewindStates.Restore(&errorString);
 				if (result == CChunkFileReader::ERROR_NONE) {
 					callbackMessage = sc->T("Loaded State");

assets/lang/az_AZ.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Savestate Slot = Savestate slot
 Savestate slot backups = Savestate slot backups
 Screenshot mode = Screenshot mode

assets/lang/be_BY.ini

@@ -1411,7 +1411,7 @@ Recording = Запіс
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Savestate Slot = Слот Savestate
 Savestate slot backups = Рэзервовыя копіі слотаў Savestate
 Screenshot mode = Рэжым скрыншота

assets/lang/bg_BG.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Възстанови първоначалните настройки на PPSSPP
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind snapshot честота („яде“ памет)
+Rewind Snapshot Interval = Rewind snapshot честота
 Savestate Slot = слот за запазено състояние
 Savestate slot backups = Savestate slot backups
 Screenshot mode = Screenshot mode

assets/lang/ca_ES.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Savestate Slot = Savestate slot
 Savestate slot backups = Savestate slot backups
 Screenshot mode = Screenshot mode

assets/lang/cz_CZ.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Obnovit výchozí nastavení PPSSPP
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Četnost snímků přetočení (žrout paměti)
+Rewind Snapshot Interval = Četnost snímků přetočení
 Savestate Slot = Pozice uložené hry
 Savestate slot backups = Savestate slot backups
 Screenshot mode = Screenshot mode

assets/lang/da_DK.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Sæt PPSSPP's indstillinger tilbage til standard
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Tilbagespol snapshot frekvens (mem hog)
+Rewind Snapshot Interval = Tilbagespol snapshot frekvens
 Savestate Slot = Lagerplads for spil-status
 Savestate slot backups = Savestate slot backups
 Screenshot mode = Screenshot mode

assets/lang/de_DE.ini

@@ -1409,7 +1409,7 @@ Recording = Aufnahme
 Reset Recording on Save/Load State = Aufnahme zurücksetzen bei Laden/Speichern eines Spielstands
 Restore Default Settings = Auf Vorgabeeinstellungen zurücksetzen
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Schnappschussintervall zurückspulen (Speicherfresser)
+Rewind Snapshot Interval = Schnappschussintervall zurückspulen
 Savestate Slot = Speicherstand-Slot
 Savestate slot backups = Backups für Speicherstände
 Screenshot mode = Bildschirmfotomodus

assets/lang/dr_ID.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Savestate Slot = Savestate slot
 Savestate slot backups = Savestate slot backups
 Screenshot mode = Screenshot mode

assets/lang/en_US.ini

@@ -1435,7 +1435,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Savestate Slot = Savestate slot
 Savestate slot backups = Savestate slot backups
 Screenshot mode = Screenshot mode

assets/lang/es_ES.ini

@@ -1422,7 +1422,7 @@ Recording = Grabación
 Reset Recording on Save/Load State = Restablecer grabación en el estado Guardar/Cargar
 Restore Default Settings = Restablecer ajustes por defecto
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Intervalo de rebobinado de instantáneas (consumo excesivo de memoria)
+Rewind Snapshot Interval = Intervalo de rebobinado de instantáneas
 Savestate Slot = Ranura para guardado de estado
 Savestate slot backups = Ranura para copias de seguridad de estado
 Screenshot mode = Modo de captura de pantalla

assets/lang/es_LA.ini

@@ -1423,7 +1423,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reiniciar grabación al abrir/guardar estados
 Restore Default Settings = Reestablecer ajustes
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Frecuencia de rebobinado\n(consume memoria)
+Rewind Snapshot Interval = Frecuencia de rebobinado
 Savestate Slot = Ranura de estado guardado
 Savestate slot backups = Copias de seguridad de estado guardado
 Screenshot mode = Screenshot mode

assets/lang/fi_FI.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Nollaa nauhoitus tallennettaessa/ladattaessa tila
 Restore Default Settings = Palauta PPSSPP:n oletusasetukset
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Pikakelaa tilannevedosten välit (muistisyöppö)
+Rewind Snapshot Interval = Pikakelaa tilannevedosten välit
 Savestate Slot = Tilatallennuksen lohko
 Savestate slot backups = Tallennustilan lohkon varmuuskopiot
 Screenshot mode = Screenshot mode

assets/lang/fr_FR.ini

@@ -1412,7 +1412,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Redémarrer l'enregistrement lors de la sauvegarde/chargement d'état
 Restore Default Settings = Restaurer les paramètres par défaut
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Fréquence instantanés rembobinage (+ de mémoire)
+Rewind Snapshot Interval = Fréquence instantanés rembobinage
 Savestate Slot = Emplacement d'état
 Savestate slot backups = Emplacement d'état de secours
 Screenshot mode = Screenshot mode

assets/lang/gl_ES.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Reestablecer axustes
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Frecuencia de rebobinado de instantánea (mem hog)
+Rewind Snapshot Interval = Frecuencia de rebobinado de instantánea
 Savestate Slot = Ranura de estado gardado
 Savestate slot backups = Savestate slot backups
 Screenshot mode = Screenshot mode

assets/lang/gr_EL.ini

@@ -1421,7 +1421,7 @@ Recording = Recording
 Reset Recording on Save/Load State = Επαναφορά της εγγραφής κατή την Αποθήκευση/Φόρτωση σημείου αποθήκευσης
 Restore Default Settings = Επαναφορά προεπιλεγμένων ρυθμίσεων του PPSSPP
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Συχνότητα Αντιστροφής Στιγμιότυπου (mem hog)
+Rewind Snapshot Interval = Συχνότητα Αντιστροφής Στιγμιότυπου
 Savestate Slot = Slot Σημείου Αποθήκευσης
 Savestate slot backups = Αντίγραφα ασφαλείας slot σημείων αποθήκευσης
 Screenshot mode = Screenshot mode