Merge pull request #6874 from Robyt3/CJsonWriter

Port JSON writer from upstream, improve testing

CMakeLists.txt

@@ -1956,6 +1956,8 @@ set_src(ENGINE_SHARED GLOB_RECURSE src/engine/shared
   jobs.h
   json.cpp
   json.h
+  jsonwriter.cpp
+  jsonwriter.h
   kernel.cpp
   linereader.cpp
   linereader.h
@@ -2726,6 +2728,7 @@ if(GTEST_FOUND OR DOWNLOAD_GTEST)
     io.cpp
     jobs.cpp
     json.cpp
+    jsonwriter.cpp
     linereader.cpp
     mapbugs.cpp
     name_ban.cpp

src/engine/shared/jsonwriter.cpp

@@ -0,0 +1,234 @@
+/* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
+/* If you are missing that file, acquire a complete release at teeworlds.com.                */
+
+#include "jsonwriter.h"
+
+static char EscapeJsonChar(char c)
+{
+	switch(c)
+	{
+	case '\"': return '\"';
+	case '\\': return '\\';
+	case '\b': return 'b';
+	case '\n': return 'n';
+	case '\r': return 'r';
+	case '\t': return 't';
+	// Don't escape '\f', who uses that. :)
+	default: return 0;
+	}
+}
+
+CJsonWriter::CJsonWriter()
+{
+	m_Indentation = 0;
+}
+
+void CJsonWriter::BeginObject()
+{
+	dbg_assert(CanWriteDatatype(), "Cannot write object here");
+	WriteIndent(false);
+	WriteInternal("{");
+	PushState(STATE_OBJECT);
+}
+
+void CJsonWriter::EndObject()
+{
+	dbg_assert(TopState()->m_Kind == STATE_OBJECT, "Cannot end object here");
+	PopState();
+	CompleteDataType();
+	WriteIndent(true);
+	WriteInternal("}");
+}
+
+void CJsonWriter::BeginArray()
+{
+	dbg_assert(CanWriteDatatype(), "Cannot write array here");
+	WriteIndent(false);
+	WriteInternal("[");
+	PushState(STATE_ARRAY);
+}
+
+void CJsonWriter::EndArray()
+{
+	dbg_assert(TopState()->m_Kind == STATE_ARRAY, "Cannot end array here");
+	PopState();
+	CompleteDataType();
+	WriteIndent(true);
+	WriteInternal("]");
+}
+
+void CJsonWriter::WriteAttribute(const char *pName)
+{
+	dbg_assert(TopState()->m_Kind == STATE_OBJECT, "Cannot write attribute here");
+	WriteIndent(false);
+	WriteInternalEscaped(pName);
+	WriteInternal(": ");
+	PushState(STATE_ATTRIBUTE);
+}
+
+void CJsonWriter::WriteStrValue(const char *pValue)
+{
+	dbg_assert(CanWriteDatatype(), "Cannot write value here");
+	WriteIndent(false);
+	WriteInternalEscaped(pValue);
+	CompleteDataType();
+}
+
+void CJsonWriter::WriteIntValue(int Value)
+{
+	dbg_assert(CanWriteDatatype(), "Cannot write value here");
+	WriteIndent(false);
+	char aBuf[32];
+	str_format(aBuf, sizeof(aBuf), "%d", Value);
+	WriteInternal(aBuf);
+	CompleteDataType();
+}
+
+void CJsonWriter::WriteBoolValue(bool Value)
+{
+	dbg_assert(CanWriteDatatype(), "Cannot write value here");
+	WriteIndent(false);
+	WriteInternal(Value ? "true" : "false");
+	CompleteDataType();
+}
+
+void CJsonWriter::WriteNullValue()
+{
+	dbg_assert(CanWriteDatatype(), "Cannot write value here");
+	WriteIndent(false);
+	WriteInternal("null");
+	CompleteDataType();
+}
+
+bool CJsonWriter::CanWriteDatatype()
+{
+	return m_States.empty() || TopState()->m_Kind == STATE_ARRAY || TopState()->m_Kind == STATE_ATTRIBUTE;
+}
+
+void CJsonWriter::WriteInternalEscaped(const char *pStr)
+{
+	WriteInternal("\"");
+	int UnwrittenFrom = 0;
+	int Length = str_length(pStr);
+	for(int i = 0; i < Length; i++)
+	{
+		char SimpleEscape = EscapeJsonChar(pStr[i]);
+		// Assuming ASCII/UTF-8, exactly everything below 0x20 is a
+		// control character.
+		bool NeedsEscape = SimpleEscape || (unsigned char)pStr[i] < 0x20;
+		if(NeedsEscape)
+		{
+			if(i - UnwrittenFrom > 0)
+			{
+				WriteInternal(pStr + UnwrittenFrom, i - UnwrittenFrom);
+			}
+
+			if(SimpleEscape)
+			{
+				char aStr[2];
+				aStr[0] = '\\';
+				aStr[1] = SimpleEscape;
+				WriteInternal(aStr, sizeof(aStr));
+			}
+			else
+			{
+				char aStr[7];
+				str_format(aStr, sizeof(aStr), "\\u%04x", pStr[i]);
+				WriteInternal(aStr);
+			}
+			UnwrittenFrom = i + 1;
+		}
+	}
+	if(Length - UnwrittenFrom > 0)
+	{
+		WriteInternal(pStr + UnwrittenFrom, Length - UnwrittenFrom);
+	}
+	WriteInternal("\"");
+}
+
+void CJsonWriter::WriteIndent(bool EndElement)
+{
+	const bool NotRootOrAttribute = !m_States.empty() && TopState()->m_Kind != STATE_ATTRIBUTE;
+
+	if(NotRootOrAttribute && !TopState()->m_Empty && !EndElement)
+		WriteInternal(",");
+
+	if(NotRootOrAttribute || EndElement)
+		WriteInternal("\n");
+
+	if(NotRootOrAttribute)
+		for(int i = 0; i < m_Indentation; i++)
+			WriteInternal("\t");
+}
+
+void CJsonWriter::PushState(EJsonStateKind NewState)
+{
+	if(!m_States.empty())
+	{
+		m_States.top().m_Empty = false;
+	}
+	m_States.push(SState(NewState));
+	if(NewState != STATE_ATTRIBUTE)
+	{
+		m_Indentation++;
+	}
+}
+
+CJsonWriter::SState *CJsonWriter::TopState()
+{
+	dbg_assert(!m_States.empty(), "json stack is empty");
+	return &m_States.top();
+}
+
+CJsonWriter::EJsonStateKind CJsonWriter::PopState()
+{
+	dbg_assert(!m_States.empty(), "json stack is empty");
+	SState TopState = m_States.top();
+	m_States.pop();
+	if(TopState.m_Kind != STATE_ATTRIBUTE)
+	{
+		m_Indentation--;
+	}
+	return TopState.m_Kind;
+}
+
+void CJsonWriter::CompleteDataType()
+{
+	if(!m_States.empty() && TopState()->m_Kind == STATE_ATTRIBUTE)
+		PopState(); // automatically complete the attribute
+
+	if(!m_States.empty())
+		TopState()->m_Empty = false;
+}
+
+CJsonFileWriter::CJsonFileWriter(IOHANDLE IO)
+{
+	dbg_assert((bool)IO, "IO handle invalid");
+	m_IO = IO;
+}
+
+CJsonFileWriter::~CJsonFileWriter()
+{
+	// Ensure newline at the end
+	WriteInternal("\n");
+	io_close(m_IO);
+}
+
+void CJsonFileWriter::WriteInternal(const char *pStr, int Length)
+{
+	io_write(m_IO, pStr, Length < 0 ? str_length(pStr) : Length);
+}
+
+void CJsonStringWriter::WriteInternal(const char *pStr, int Length)
+{
+	dbg_assert(!m_RetrievedOutput, "Writer output has already been retrieved");
+	m_OutputString += Length < 0 ? pStr : std::string(pStr, Length);
+}
+
+std::string &&CJsonStringWriter::GetOutputString()
+{
+	// Ensure newline at the end. Modify member variable so we can move it when returning.
+	WriteInternal("\n");
+	m_RetrievedOutput = true; // prevent further usage of this writer
+	return std::move(m_OutputString);
+}

src/engine/shared/jsonwriter.h

@@ -0,0 +1,117 @@
+/* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
+/* If you are missing that file, acquire a complete release at teeworlds.com.                */
+#ifndef ENGINE_SHARED_JSONWRITER_H
+#define ENGINE_SHARED_JSONWRITER_H
+
+#include <base/system.h>
+
+#include <stack>
+
+/**
+ * JSON writer with abstract writing function.
+ */
+class CJsonWriter
+{
+	enum EJsonStateKind
+	{
+		STATE_OBJECT,
+		STATE_ARRAY,
+		STATE_ATTRIBUTE,
+	};
+
+	struct SState
+	{
+		EJsonStateKind m_Kind;
+		bool m_Empty = true;
+
+		SState(EJsonStateKind Kind) :
+			m_Kind(Kind)
+		{
+		}
+	};
+
+	std::stack<SState> m_States;
+	int m_Indentation;
+
+	bool CanWriteDatatype();
+	void WriteInternalEscaped(const char *pStr);
+	void WriteIndent(bool EndElement);
+	void PushState(EJsonStateKind NewState);
+	SState *TopState();
+	EJsonStateKind PopState();
+	void CompleteDataType();
+
+protected:
+	// String must be zero-terminated when Length is -1.
+	virtual void WriteInternal(const char *pStr, int Length = -1) = 0;
+
+public:
+	CJsonWriter();
+	virtual ~CJsonWriter() = default;
+
+	// The root is created by beginning the first datatype (object, array, value).
+	// The writer must not be used after ending the root, which must be unique.
+
+	// Begin writing a new object
+	void BeginObject();
+	// End current object
+	void EndObject();
+
+	// Begin writing a new array
+	void BeginArray();
+	// End current array
+	void EndArray();
+
+	// Write attribute with the given name inside the current object.
+	// Names inside one object should be unique, but this is not checked here.
+	// Must be used to begin writing anything inside objects and only there.
+	// Must be followed by a datatype for the attribute value.
+	void WriteAttribute(const char *pName);
+
+	// Functions for writing value literals:
+	// - As array values in arrays.
+	// - As attribute values after beginning an attribute inside an object.
+	// - As root value (only once).
+	void WriteStrValue(const char *pValue);
+	void WriteIntValue(int Value);
+	void WriteBoolValue(bool Value);
+	void WriteNullValue();
+};
+
+/**
+ * Writes JSON to a file.
+ */
+class CJsonFileWriter : public CJsonWriter
+{
+	IOHANDLE m_IO;
+
+protected:
+	void WriteInternal(const char *pStr, int Length = -1) override;
+
+public:
+	/**
+	 * Create a new writer object without writing anything to the file yet.
+	 * The file will automatically be closed by the destructor.
+	 */
+	CJsonFileWriter(IOHANDLE IO);
+	~CJsonFileWriter();
+};
+
+/**
+ * Writes JSON to an std::string.
+ */
+class CJsonStringWriter : public CJsonWriter
+{
+	std::string m_OutputString;
+	bool m_RetrievedOutput = false;
+
+protected:
+	void WriteInternal(const char *pStr, int Length = -1) override;
+
+public:
+	CJsonStringWriter() = default;
+	~CJsonStringWriter() = default;
+	std::string &&GetOutputString();
+};
+
+#endif

src/test/jsonwriter.cpp

@@ -0,0 +1,211 @@
+/* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
+/* If you are missing that file, acquire a complete release at teeworlds.com.                */
+#include "test.h"
+#include <gtest/gtest.h>
+
+#include <engine/shared/jsonwriter.h>
+
+#include <climits>
+
+class JsonFileWriter
+{
+public:
+	CTestInfo m_Info;
+	CJsonFileWriter *m_pJson;
+	char m_aOutputFilename[IO_MAX_PATH_LENGTH];
+
+	JsonFileWriter() :
+		m_pJson(nullptr)
+	{
+		m_Info.Filename(m_aOutputFilename, sizeof(m_aOutputFilename), "-got.json");
+		IOHANDLE File = io_open(m_aOutputFilename, IOFLAG_WRITE);
+		EXPECT_TRUE(File);
+		m_pJson = new CJsonFileWriter(File);
+	}
+
+	void Expect(const char *pExpected)
+	{
+		ASSERT_TRUE(m_pJson);
+		delete m_pJson;
+		m_pJson = nullptr;
+
+		IOHANDLE GotFile = io_open(m_aOutputFilename, IOFLAG_READ);
+		ASSERT_TRUE(GotFile);
+		char *pOutput = io_read_all_str(GotFile);
+		io_close(GotFile);
+		ASSERT_TRUE(pOutput);
+		EXPECT_STREQ(pOutput, pExpected);
+		bool Correct = str_comp(pOutput, pExpected) == 0;
+		free(pOutput);
+
+		if(!Correct)
+		{
+			char aFilename[IO_MAX_PATH_LENGTH];
+			m_Info.Filename(aFilename, sizeof(aFilename), "-expected.json");
+			IOHANDLE ExpectedFile = io_open(aFilename, IOFLAG_WRITE);
+			ASSERT_TRUE(ExpectedFile);
+			io_write(ExpectedFile, pExpected, str_length(pExpected));
+			io_close(ExpectedFile);
+		}
+		else
+		{
+			fs_remove(m_aOutputFilename);
+		}
+	}
+};
+
+class JsonStringWriter
+{
+public:
+	CJsonStringWriter *m_pJson;
+
+	JsonStringWriter() :
+		m_pJson(nullptr)
+	{
+		m_pJson = new CJsonStringWriter();
+	}
+
+	void Expect(const char *pExpected)
+	{
+		ASSERT_TRUE(m_pJson);
+		const std::string OutputString = m_pJson->GetOutputString();
+		EXPECT_STREQ(OutputString.c_str(), pExpected);
+		delete m_pJson;
+		m_pJson = nullptr;
+	}
+};
+
+template<typename T>
+class JsonWriters : public testing::Test
+{
+public:
+	T Impl;
+};
+
+using JsonWriterTestFixures = ::testing::Types<JsonFileWriter, JsonStringWriter>;
+TYPED_TEST_SUITE(JsonWriters, JsonWriterTestFixures);
+
+TYPED_TEST(JsonWriters, Empty)
+{
+	this->Impl.Expect("\n");
+}
+
+TYPED_TEST(JsonWriters, EmptyObject)
+{
+	this->Impl.m_pJson->BeginObject();
+	this->Impl.m_pJson->EndObject();
+	this->Impl.Expect("{\n}\n");
+}
+
+TYPED_TEST(JsonWriters, EmptyArray)
+{
+	this->Impl.m_pJson->BeginArray();
+	this->Impl.m_pJson->EndArray();
+	this->Impl.Expect("[\n]\n");
+}
+
+TYPED_TEST(JsonWriters, SpecialCharacters)
+{
+	this->Impl.m_pJson->BeginObject();
+	this->Impl.m_pJson->WriteAttribute("\x01\"'\r\n\t");
+	this->Impl.m_pJson->BeginArray();
+	this->Impl.m_pJson->WriteStrValue(" \"'abc\x01\n");
+	this->Impl.m_pJson->EndArray();
+	this->Impl.m_pJson->EndObject();
+	this->Impl.Expect(
+		"{\n"
+		"\t\"\\u0001\\\"'\\r\\n\\t\": [\n"
+		"\t\t\" \\\"'abc\\u0001\\n\"\n"
+		"\t]\n"
+		"}\n");
+}
+
+TYPED_TEST(JsonWriters, HelloWorld)
+{
+	this->Impl.m_pJson->WriteStrValue("hello world");
+	this->Impl.Expect("\"hello world\"\n");
+}
+
+TYPED_TEST(JsonWriters, Unicode)
+{
+	this->Impl.m_pJson->WriteStrValue("Heizölrückstoßabdämpfung");
+	this->Impl.Expect("\"Heizölrückstoßabdämpfung\"\n");
+}
+
+TYPED_TEST(JsonWriters, True)
+{
+	this->Impl.m_pJson->WriteBoolValue(true);
+	this->Impl.Expect("true\n");
+}
+
+TYPED_TEST(JsonWriters, False)
+{
+	this->Impl.m_pJson->WriteBoolValue(false);
+	this->Impl.Expect("false\n");
+}
+
+TYPED_TEST(JsonWriters, Null)
+{
+	this->Impl.m_pJson->WriteNullValue();
+	this->Impl.Expect("null\n");
+}
+
+TYPED_TEST(JsonWriters, EmptyString)
+{
+	this->Impl.m_pJson->WriteStrValue("");
+	this->Impl.Expect("\"\"\n");
+}
+
+TYPED_TEST(JsonWriters, EscapeNewline)
+{
+	this->Impl.m_pJson->WriteStrValue("\n");
+	this->Impl.Expect("\"\\n\"\n");
+}
+
+TYPED_TEST(JsonWriters, EscapeBackslash)
+{
+	this->Impl.m_pJson->WriteStrValue("\\");
+	this->Impl.Expect("\"\\\\\"\n"); // https://www.xkcd.com/1638/
+}
+
+TYPED_TEST(JsonWriters, EscapeControl)
+{
+	this->Impl.m_pJson->WriteStrValue("\x1b");
+	this->Impl.Expect("\"\\u001b\"\n");
+}
+
+TYPED_TEST(JsonWriters, EscapeUnicode)
+{
+	this->Impl.m_pJson->WriteStrValue("愛😂");
+	this->Impl.Expect("\"愛😂\"\n");
+}
+
+TYPED_TEST(JsonWriters, Zero)
+{
+	this->Impl.m_pJson->WriteIntValue(0);
+	this->Impl.Expect("0\n");
+}
+
+TYPED_TEST(JsonWriters, One)
+{
+	this->Impl.m_pJson->WriteIntValue(1);
+	this->Impl.Expect("1\n");
+}
+
+TYPED_TEST(JsonWriters, MinusOne)
+{
+	this->Impl.m_pJson->WriteIntValue(-1);
+	this->Impl.Expect("-1\n");
+}
+
+TYPED_TEST(JsonWriters, Large)
+{
+	this->Impl.m_pJson->WriteIntValue(INT_MAX);
+	this->Impl.Expect("2147483647\n");
+}
+
+TYPED_TEST(JsonWriters, Small)
+{
+	this->Impl.m_pJson->WriteIntValue(INT_MIN);
+	this->Impl.Expect("-2147483648\n");
+}

src/test/test.cpp

@@ -11,9 +11,29 @@ CTestInfo::CTestInfo()
 {
 	const ::testing::TestInfo *pTestInfo =
 		::testing::UnitTest::GetInstance()->current_test_info();
-	char aBuf[IO_MAX_PATH_LENGTH];
-	str_format(aBuf, sizeof(aBuf), "%s.%s", pTestInfo->test_case_name(), pTestInfo->name());
-	IStorage::FormatTmpPath(m_aFilename, sizeof(m_aFilename), aBuf);
+
+	// Typed tests have test names like "TestName/0" and "TestName/1", which would result in invalid filenames.
+	// Replace the string after the first slash with the name of the typed test and use hyphen instead of slash.
+	char aTestCaseName[128];
+	str_copy(aTestCaseName, pTestInfo->test_case_name());
+	for(int i = 0; i < str_length(aTestCaseName); i++)
+	{
+		if(aTestCaseName[i] == '/')
+		{
+			aTestCaseName[i] = '-';
+			aTestCaseName[i + 1] = '\0';
+			str_append(aTestCaseName, pTestInfo->type_param());
+			break;
+		}
+	}
+	str_format(m_aFilenamePrefix, sizeof(m_aFilenamePrefix), "%s.%s-%d",
+		aTestCaseName, pTestInfo->name(), pid());
+	Filename(m_aFilename, sizeof(m_aFilename), ".tmp");
+}
+
+void CTestInfo::Filename(char *pBuffer, size_t BufferLength, const char *pSuffix)
+{
+	str_format(pBuffer, BufferLength, "%s%s", m_aFilenamePrefix, pSuffix);
 }
 
 IStorage *CTestInfo::CreateTestStorage()

src/test/test.h

@@ -1,6 +1,8 @@
 #ifndef TEST_TEST_H
 #define TEST_TEST_H
 
+#include <cstddef>
+
 class IStorage;
 
 class CTestInfo
@@ -10,6 +12,8 @@ public:
 	~CTestInfo();
 	IStorage *CreateTestStorage();
 	bool m_DeleteTestStorageFilesOnSuccess = false;
-	char m_aFilename[64];
+	void Filename(char *pBuffer, size_t BufferLength, const char *pSuffix);
+	char m_aFilenamePrefix[128];
+	char m_aFilename[128];
 };
 #endif // TEST_TEST_H