Improve Android storage usage, faster launch, remove permissions

Split the user storage location and the data folder in the app specific external storage in the folders `data` and `user` instead of writing the user setting directly to the external storage.

Remove unnecessary storage permissions. The client only accesses files in its own external storage location, hence these permissions are not necessary for Android API 19 and higher, which is always given as we only target API 19 and higher.

Only unpack changed assets when their hash in the integrity index is different instead of unpacking all assets again, so the app starts faster after updates. Avoid unpacking the entire integrity index file unless it changed, by initially reading only the first hash directly from the asset, so the app starts faster when the data is up-to-date.

Add error handling for external storage not being accessible and other I/O errors during unpacking of assets.

Add `android_main.h` header to export the `InitAndroid` function and potentially other functions in the future. The `extern "C"` and `__attribute__((visibility("default")))` attributes seem to be unnecessary, as this function is only called directly from the native code like many other functions without these attributes.

Initialize the Android storage after the loggers, so the log message are printed properly.

Add documentation for the use of `std::exit` on Android, which is used to forcefully terminate the entire process, to ensure that static variables will be initialized correctly when the app is started again after quitting. Returning from the main function is not enough, as this only results in the native thread terminating, but the Java thread will continue. Java does not support unloading libraries once they have been loaded, so all static variables will not have their expected initial values anymore when the app is started again after quitting.

Use `fs_chdir` and `fs_makedir` instead of `chdir` and `mkdir`.

scripts/android/files/AndroidManifest.xml

@@ -9,10 +9,6 @@
 	<uses-permission android:name="android.permission.ACCESS_WIFI_STATE"/>
 	<uses-permission android:name="android.permission.CHANGE_WIFI_MULTICAST_STATE"/>
 	<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
-
-	<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
-	<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
-	
 	<uses-permission android:name="android.permission.INTERNET" />
 
 	<!-- usesCleartextTraffic because unencrypted UDP packets -->

src/android/android_main.cpp

@@ -1,160 +1,231 @@
-#include <base/detect.h>
+#include "android_main.h"
-
-#ifdef CONF_PLATFORM_ANDROID
-#include <sys/stat.h>
-#include <unistd.h>
 
 #include <SDL.h>
 
 #include <base/hash.h>
+#include <base/log.h>
 #include <base/system.h>
+
 #include <engine/shared/linereader.h>
+
 #include <string>
 #include <vector>
 
-extern "C" __attribute__((visibility("default"))) void InitAndroid();
+static bool UnpackAsset(const char *pFilename)
-
-static int gs_AndroidStarted = false;
-
-void InitAndroid()
 {
-	if(gs_AndroidStarted)
+	char aAssetFilename[IO_MAX_PATH_LENGTH];
+	str_copy(aAssetFilename, "asset_integrity_files/");
+	str_append(aAssetFilename, pFilename);
+
+	// This uses SDL_RWFromFile because it can read Android assets,
+	// which are files stored in the app's APK file. All data files
+	// are stored as assets and unpacked to the external storage.
+	SDL_RWops *pAssetFile = SDL_RWFromFile(aAssetFilename, "rb");
+	if(!pAssetFile)
 	{
-		SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_ERROR, "DDNet", "The app was started, but not closed properly, this causes bugs. Please restart or manually delete this task.", SDL_GL_GetCurrentWindow());
+		log_error("android", "Failed to open asset '%s' for reading", pFilename);
-		std::exit(0);
+		return false;
 	}
 
-	gs_AndroidStarted = true;
+	const long int FileLength = SDL_RWsize(pAssetFile);
-
+	if(FileLength < 0)
-	// change current path to a writable directory
-	const char *pPath = SDL_AndroidGetExternalStoragePath();
-	chdir(pPath);
-	dbg_msg("client", "changed path to %s", pPath);
-
-	// copy integrity files
 	{
-		SDL_RWops *pF = SDL_RWFromFile("asset_integrity_files/integrity.txt", "rb");
+		SDL_RWclose(pAssetFile);
-		if(!pF)
+		log_error("android", "Failed to determine length of asset '%s'", pFilename);
-		{
+		return false;
-			SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_ERROR, "DDNet", "integrity.txt not found, consider reinstalling", SDL_GL_GetCurrentWindow());
-			std::exit(0);
-		}
-
-		long int length;
-		SDL_RWseek(pF, 0, RW_SEEK_END);
-		length = SDL_RWtell(pF);
-		SDL_RWseek(pF, 0, RW_SEEK_SET);
-
-		char *pAl = (char *)malloc(length);
-		SDL_RWread(pF, pAl, 1, length);
-
-		SDL_RWclose(pF);
-
-		mkdir("data", 0755);
-
-		dbg_msg("integrity", "copying integrity.txt with size: %ld", length);
-
-		IOHANDLE IntegrityFileWrite = io_open("integrity.txt", IOFLAG_WRITE);
-		io_write(IntegrityFileWrite, pAl, length);
-		io_close(IntegrityFileWrite);
-
-		free(pAl);
 	}
 
-	IOHANDLE IntegrityFileRead = io_open("integrity.txt", IOFLAG_READ);
+	char *pData = static_cast<char *>(malloc(FileLength));
-	CLineReader IntegrityFileLineReader;
+	const size_t ReadLength = SDL_RWread(pAssetFile, pData, 1, FileLength);
-	IntegrityFileLineReader.Init(IntegrityFileRead);
+	SDL_RWclose(pAssetFile);
-	const char *pReadLine = NULL;
-	std::vector<std::string> vLines;
-	while((pReadLine = IntegrityFileLineReader.Get()))
-	{
-		vLines.push_back(pReadLine);
-	}
-	io_close(IntegrityFileRead);
 
-	// first line is the whole hash
+	if(ReadLength != (size_t)FileLength)
-	std::string AllAsOne;
-	for(size_t i = 1; i < vLines.size(); ++i)
 	{
-		AllAsOne.append(vLines[i]);
+		free(pData);
-		AllAsOne.append("\n");
+		log_error("android", "Failed to read asset '%s' (read %" PRIzu ", wanted %ld)", pFilename, ReadLength, FileLength);
-	}
+		return false;
-	SHA256_DIGEST ShaAll;
-	bool GotSHA = false;
-	{
-		IOHANDLE IntegritySaveFileRead = io_open("integrity_save.txt", IOFLAG_READ);
-		if(IntegritySaveFileRead != NULL)
-		{
-			CLineReader IntegritySaveLineReader;
-			IntegritySaveLineReader.Init(IntegritySaveFileRead);
-			const char *pLine = IntegritySaveLineReader.Get();
-			if(pLine != NULL)
-			{
-				sha256_from_str(&ShaAll, pLine);
-				GotSHA = true;
-			}
-			io_close(IntegritySaveFileRead);
-		}
 	}
 
-	SHA256_DIGEST ShaAllFile;
+	IOHANDLE TargetFile = io_open(pFilename, IOFLAG_WRITE);
-	sha256_from_str(&ShaAllFile, vLines[0].c_str());
+	if(!TargetFile)
-
-	// TODO: check files individually
-	if(!GotSHA || ShaAllFile != ShaAll)
 	{
-		// then the files
+		free(pData);
-		for(size_t i = 1; i < vLines.size(); ++i)
+		log_error("android", "Failed to open '%s' for writing", pFilename);
-		{
+		return false;
-			std::string FileName, Hash;
-			std::string::size_type n = 0;
-			std::string::size_type c = 0;
-			while((c = vLines[i].find(' ', n)) != std::string::npos)
-				n = c + 1;
-			FileName = vLines[i].substr(0, n - 1);
-			Hash = vLines[i].substr(n + 1);
-
-			std::string AssetFileName = std::string("asset_integrity_files/") + FileName;
-			SDL_RWops *pF = SDL_RWFromFile(AssetFileName.c_str(), "rb");
-
-			dbg_msg("Integrity", "Copying from assets: %s", FileName.c_str());
-
-			std::string FileNamePath = FileName;
-			std::string FileNamePathSub;
-			c = 0;
-			while((c = FileNamePath.find('/', c)) != std::string::npos)
-			{
-				FileNamePathSub = FileNamePath.substr(0, c);
-				fs_makedir(FileNamePathSub.c_str());
-				++c;
-			}
-
-			long int length;
-			SDL_RWseek(pF, 0, RW_SEEK_END);
-			length = SDL_RWtell(pF);
-			SDL_RWseek(pF, 0, RW_SEEK_SET);
-
-			char *pAl = (char *)malloc(length);
-			SDL_RWread(pF, pAl, 1, length);
-
-			SDL_RWclose(pF);
-
-			IOHANDLE AssetFileWrite = io_open(FileName.c_str(), IOFLAG_WRITE);
-			io_write(AssetFileWrite, pAl, length);
-			io_close(AssetFileWrite);
-
-			free(pAl);
-		}
-
-		IOHANDLE IntegritySaveFileWrite = io_open("integrity_save.txt", IOFLAG_WRITE);
-		if(IntegritySaveFileWrite != NULL)
-		{
-			char aFileSHA[SHA256_MAXSTRSIZE];
-			sha256_str(ShaAllFile, aFileSHA, sizeof(aFileSHA));
-			io_write(IntegritySaveFileWrite, aFileSHA, str_length(aFileSHA));
-			io_close(IntegritySaveFileWrite);
-		}
 	}
+
+	const size_t WriteLength = io_write(TargetFile, pData, FileLength);
+	io_close(TargetFile);
+	free(pData);
+
+	if(WriteLength != (size_t)FileLength)
+	{
+		log_error("android", "Failed to write data to '%s' (wrote %" PRIzu ", wanted %ld)", pFilename, WriteLength, FileLength);
+		return false;
+	}
+
+	return true;
 }
 
-#endif
+constexpr const char *INTEGRITY_INDEX = "integrity.txt";
+constexpr const char *INTEGRITY_INDEX_SAVE = "integrity_save.txt";
+
+// The first line of each integrity file contains the combined hash for all files,
+// if the hashes match then we assume that the unpacked data folder is up-to-date.
+static bool EqualIntegrityFiles(const char *pAssetFilename, const char *pStorageFilename)
+{
+	IOHANDLE StorageFile = io_open(pStorageFilename, IOFLAG_READ);
+	if(!StorageFile)
+	{
+		return false;
+	}
+
+	char aStorageMainSha256[SHA256_MAXSTRSIZE];
+	const size_t StorageReadLength = io_read(StorageFile, aStorageMainSha256, sizeof(aStorageMainSha256) - 1);
+	io_close(StorageFile);
+	if(StorageReadLength != sizeof(aStorageMainSha256) - 1)
+	{
+		return false;
+	}
+	aStorageMainSha256[sizeof(aStorageMainSha256) - 1] = '\0';
+
+	char aAssetFilename[IO_MAX_PATH_LENGTH];
+	str_copy(aAssetFilename, "asset_integrity_files/");
+	str_append(aAssetFilename, pAssetFilename);
+
+	SDL_RWops *pAssetFile = SDL_RWFromFile(aAssetFilename, "rb");
+	if(!pAssetFile)
+	{
+		return false;
+	}
+
+	char aAssetMainSha256[SHA256_MAXSTRSIZE];
+	const size_t AssetReadLength = SDL_RWread(pAssetFile, aAssetMainSha256, 1, sizeof(aAssetMainSha256) - 1);
+	SDL_RWclose(pAssetFile);
+	if(AssetReadLength != sizeof(aAssetMainSha256) - 1)
+	{
+		return false;
+	}
+	aAssetMainSha256[sizeof(aAssetMainSha256) - 1] = '\0';
+
+	return str_comp(aStorageMainSha256, aAssetMainSha256) == 0;
+}
+
+class CIntegrityFileLine
+{
+public:
+	char m_aFilename[IO_MAX_PATH_LENGTH];
+	SHA256_DIGEST m_Sha256;
+};
+
+static std::vector<CIntegrityFileLine> ReadIntegrityFile(const char *pFilename)
+{
+	IOHANDLE IntegrityFile = io_open(pFilename, IOFLAG_READ);
+	if(!IntegrityFile)
+	{
+		return {};
+	}
+
+	CLineReader LineReader;
+	LineReader.Init(IntegrityFile);
+	const char *pReadLine;
+	std::vector<CIntegrityFileLine> vLines;
+	while((pReadLine = LineReader.Get()))
+	{
+		const char *pSpaceInLine = str_rchr(pReadLine, ' ');
+		CIntegrityFileLine Line;
+		char aSha256[SHA256_MAXSTRSIZE];
+		if(pSpaceInLine == nullptr)
+		{
+			if(!vLines.empty())
+			{
+				// Only the first line is allowed to not contain a filename
+				log_error("android", "Failed to parse line %" PRIzu " of '%s': line does not contain space", vLines.size() + 1, pFilename);
+				return {};
+			}
+			Line.m_aFilename[0] = '\0';
+			str_copy(aSha256, pReadLine);
+		}
+		else
+		{
+			str_truncate(Line.m_aFilename, sizeof(Line.m_aFilename), pReadLine, pSpaceInLine - pReadLine);
+			str_copy(aSha256, pSpaceInLine + 1);
+		}
+		if(sha256_from_str(&Line.m_Sha256, aSha256) != 0)
+		{
+			log_error("android", "Failed to parse line %" PRIzu " of '%s': invalid SHA256 string", vLines.size() + 1, pFilename);
+			return {};
+		}
+		vLines.emplace_back(std::move(Line));
+	}
+
+	io_close(IntegrityFile);
+	return vLines;
+}
+
+const char *InitAndroid()
+{
+	// Change current working directory to our external storage location
+	const char *pPath = SDL_AndroidGetExternalStoragePath();
+	if(pPath == nullptr)
+	{
+		return "The external storage is not available.";
+	}
+	if(fs_chdir(pPath) != 0)
+	{
+		return "Failed to change current directory to external storage.";
+	}
+	log_info("android", "Changed current directory to '%s'", pPath);
+
+	if(fs_makedir("data") != 0 || fs_makedir("user") != 0)
+	{
+		return "Failed to create 'data' and 'user' directories in external storage.";
+	}
+
+	if(EqualIntegrityFiles(INTEGRITY_INDEX, INTEGRITY_INDEX_SAVE))
+	{
+		return nullptr;
+	}
+
+	if(!UnpackAsset(INTEGRITY_INDEX))
+	{
+		return "Failed to unpack the integrity index file. Consider reinstalling the app.";
+	}
+
+	std::vector<CIntegrityFileLine> vIntegrityLines = ReadIntegrityFile(INTEGRITY_INDEX);
+	if(vIntegrityLines.empty())
+	{
+		return "Failed to load the integrity index file. Consider reinstalling the app.";
+	}
+
+	std::vector<CIntegrityFileLine> vIntegritySaveLines = ReadIntegrityFile(INTEGRITY_INDEX_SAVE);
+
+	// The remaining lines of each integrity file list all assets and their hashes
+	for(size_t i = 1; i < vIntegrityLines.size(); ++i)
+	{
+		const CIntegrityFileLine &IntegrityLine = vIntegrityLines[i];
+
+		// Check if the asset is unchanged from the last unpacking
+		const auto IntegritySaveLine = std::find_if(vIntegritySaveLines.begin(), vIntegritySaveLines.end(), [&](const CIntegrityFileLine &Line) {
+			return str_comp(Line.m_aFilename, IntegrityLine.m_aFilename) == 0;
+		});
+		if(IntegritySaveLine != vIntegritySaveLines.end() && IntegritySaveLine->m_Sha256 == IntegrityLine.m_Sha256)
+		{
+			continue;
+		}
+
+		if(fs_makedir_rec_for(IntegrityLine.m_aFilename) != 0 || !UnpackAsset(IntegrityLine.m_aFilename))
+		{
+			return "Failed to unpack game assets, consider reinstalling the app.";
+		}
+	}
+
+	// The integrity file will be unpacked every time when launching,
+	// so we can simply rename it to update the saved integrity file.
+	if((fs_is_file(INTEGRITY_INDEX_SAVE) && fs_remove(INTEGRITY_INDEX_SAVE) != 0) || fs_rename(INTEGRITY_INDEX, INTEGRITY_INDEX_SAVE) != 0)
+	{
+		return "Failed to update the saved integrity index file.";
+	}
+
+	return nullptr;
+}

src/android/android_main.h

@@ -0,0 +1,23 @@
+#ifndef ANDROID_ANDROID_MAIN_H
+#define ANDROID_ANDROID_MAIN_H
+
+#include <base/detect.h>
+#if !defined(CONF_PLATFORM_ANDROID)
+#error "This header should only be included when compiling for Android"
+#endif
+
+/**
+ * Initializes the Android storage. Must be called on Android-systems
+ * before using any of the I/O and storage functions.
+ *
+ * This will change the current working directory to the app specific external
+ * storage location and unpack the assets from the APK file to the `data` folder.
+ * The folder `user` is created in the external storage to store the user data.
+ *
+ * Failure must be handled by exiting the app.
+ *
+ * @return `nullptr` on success, error message on failure.
+ */
+const char *InitAndroid();
+
+#endif // ANDROID_ANDROID_MAIN_H

src/engine/client/client.cpp

@@ -55,6 +55,10 @@
 #include "video.h"
 #endif
 
+#if defined(CONF_PLATFORM_ANDROID)
+#include <android/android_main.h>
+#endif
+
 #include "SDL.h"
 #ifdef main
 #undef main
@@ -4262,9 +4266,8 @@ static void ShowMessageBox(const char *pTitle, const char *pMessage, IClient::EM
 #if defined(CONF_PLATFORM_MACOS)
 extern "C" int TWMain(int argc, const char **argv)
 #elif defined(CONF_PLATFORM_ANDROID)
+static int gs_AndroidStarted = false;
 extern "C" __attribute__((visibility("default"))) int SDL_main(int argc, char *argv[]);
-extern "C" void InitAndroid();
-
 int SDL_main(int argc, char *argv2[])
 #else
 int main(int argc, const char **argv)
@@ -4274,15 +4277,19 @@ int main(int argc, const char **argv)
 
 #if defined(CONF_PLATFORM_ANDROID)
 	const char **argv = const_cast<const char **>(argv2);
+	// Android might not unload the library from memory, causing globals like gs_AndroidStarted
+	// not to be initialized correctly when starting the app again.
+	if(gs_AndroidStarted)
+	{
+		::ShowMessageBox("Android Error", "The app was started, but not closed properly, this causes bugs. Please restart or manually close this task.");
+		std::exit(0);
+	}
+	gs_AndroidStarted = true;
 #elif defined(CONF_FAMILY_WINDOWS)
 	CWindowsComLifecycle WindowsComLifecycle(true);
 #endif
 	CCmdlineFix CmdlineFix(&argc, &argv);
 
-#if defined(CONF_PLATFORM_ANDROID)
-	InitAndroid();
-#endif
-
 #if defined(CONF_EXCEPTION_HANDLING)
 	init_exception_handler();
 #endif
@@ -4317,6 +4324,17 @@ int main(int argc, const char **argv)
 	vpLoggers.push_back(pFutureAssertionLogger);
 	log_set_global_logger(log_logger_collection(std::move(vpLoggers)).release());
 
+#if defined(CONF_PLATFORM_ANDROID)
+	// Initialize Android after logger is available
+	const char *pAndroidInitError = InitAndroid();
+	if(pAndroidInitError != nullptr)
+	{
+		log_error("android", "%s", pAndroidInitError);
+		::ShowMessageBox("Android Error", pAndroidInitError);
+		std::exit(0);
+	}
+#endif
+
 	std::stack<std::function<void()>> CleanerFunctions;
 	std::function<void()> PerformCleanup = [&CleanerFunctions]() mutable {
 		while(!CleanerFunctions.empty())
@@ -4327,7 +4345,17 @@ int main(int argc, const char **argv)
 	};
 	std::function<void()> PerformFinalCleanup = []() {
 #ifdef CONF_PLATFORM_ANDROID
-		// properly close this native thread, so globals are destructed
+		// Forcefully terminate the entire process, to ensure that static variables
+		// will be initialized correctly when the app is started again after quitting.
+		// Returning from the main function is not enough, as this only results in the
+		// native thread terminating, but the Java thread will continue. Java does not
+		// support unloading libraries once they have been loaded, so all static
+		// variables will not have their expected initial values anymore when the app
+		// is started again after quitting. The variable gs_AndroidStarted above is
+		// used to check that static variables have been initialized properly.
+		// TODO: This is not the correct way to close an activity on Android, as it
+		//       ignores the activity lifecycle entirely, which may cause issues if
+		//       we ever used any global resources like the camera.
 		std::exit(0);
 #endif
 	};

src/engine/shared/storage.cpp

@@ -39,8 +39,13 @@ public:
 
 	int Init(int StorageType, int NumArgs, const char **ppArguments)
 	{
-#if !defined(CONF_PLATFORM_ANDROID)
+#if defined(CONF_PLATFORM_ANDROID)
-		// get userdir, just use data directory on android
+		// See InitAndroid in android_main.cpp for details about Android storage handling.
+		// The current working directory is set to the app specific external storage location
+		// on Android. The user data is stored within a folder "user" in the external storage.
+		str_copy(m_aUserdir, "user");
+#else
+		// get userdir
 		char aFallbackUserdir[IO_MAX_PATH_LENGTH];
 		if(fs_storage_path("DDNet", m_aUserdir, sizeof(m_aUserdir)))
 		{