Merge #4236

4236: Reduce cpu usage from large numbers of entities + EntityEx r=def- a=trml

An attempt to fix #4233.

The problem was (as guessed) that the contains a very large number of laser doors, which each has an EntityEx. Calling SnapFindItem (which does a linear search) for each one of these turned out to be extremely slow (this was done to associate the EntityEx with the corresponding snap item), so I tried a faster way of matching them, by going through the entire snapshot first and then sorting by ID, which should be n log n instead of n^2 (this is also only done once for each snapshot). Also added network clipping, which made the cpu usage go back to normal when not zooming out.

Cpu usage now seems to be comparable with what it was on this map (also when zooming out), but it could hopefully be reduced further by cherry-picking teeworlds/teeworlds#2129, or something similar.

I also noticed a second problem, that the number of items on this map is so huge that some entities (including players) gets dropped from the snap when zooming completely out (I believe this already happened to some degree before), and the camera would sometimes revert when spectating while zoomed (which didn't seem to happen before). The last problem appeared to be fixed by snapping characters before other items.

## Checklist

- [ ] Tested the change ingame
- [ ] Provided screenshots if it is a visual change
- [ ] Tested in combination with possibly related configuration options
- [ ] Written a unit test if it works standalone, system.c especially
- [ ] Considered possible null pointers and out of bounds array indexing
- [ ] Changed no physics that affect existing maps
- [ ] Tested the change with [ASan+UBSan or valgrind's memcheck](https://github.com/ddnet/ddnet/#using-addresssanitizer--undefinedbehavioursanitizer-or-valgrinds-memcheck) (optional)


Co-authored-by: trml <trml@users.noreply.github.com>

src/game/client/components/items.cpp

@@ -350,12 +350,11 @@ void CItems::OnRender()
 		}
 	}
 
-	int Num = Client()->SnapNumItems(IClient::SNAP_CURRENT);
-	for(int i = 0; i < Num; i++)
+	for(const CSnapEntities &Ent : m_pClient->SnapEntities())
 	{
-		IClient::CSnapItem Item;
-		const void *pData = Client()->SnapGetItem(IClient::SNAP_CURRENT, i, &Item);
-		CNetObj_EntityEx *pEntEx = (CNetObj_EntityEx *)Client()->SnapFindItem(IClient::SNAP_CURRENT, NETOBJTYPE_ENTITYEX, Item.m_ID);
+		const IClient::CSnapItem Item = Ent.m_Item;
+		const void *pData = Ent.m_pData;
+		const CNetObj_EntityEx *pEntEx = Ent.m_pDataEx;
 
 		bool Inactive = false;
 		if(pEntEx)
@@ -443,6 +442,8 @@ void CItems::OnRender()
 		}
 	}
 
+	int Num = Client()->SnapNumItems(IClient::SNAP_CURRENT);
+
 	// render flag
 	for(int i = 0; i < Num; i++)
 	{

src/game/client/gameclient.cpp

@@ -1701,6 +1701,8 @@ void CGameClient::OnNewSnapshot()
 	PrevLocalID = m_Snap.m_LocalClientID;
 	m_IsDummySwapping = 0;
 
+	SnapCollectEntities(); // creates a collection that associates EntityEx snap items with the entities they belong to
+
 	// update prediction data
 	if(Client()->State() != IClient::STATE_DEMOPLAYBACK)
 		UpdatePrediction();
@@ -2426,7 +2428,6 @@ void CGameClient::UpdatePrediction()
 	m_GameWorld.m_Teams = m_Teams;
 
 	m_GameWorld.NetObjBegin();
-	int Num = Client()->SnapNumItems(IClient::SNAP_CURRENT);
 	for(int i = 0; i < MAX_CLIENTS; i++)
 		if(m_Snap.m_aCharacters[i].m_Active)
 		{
@@ -2437,13 +2438,9 @@ void CGameClient::UpdatePrediction()
 				GameTeam, IsLocal);
 		}
 
-	for(int Index = 0; Index < Num; Index++)
-	{
-		IClient::CSnapItem Item;
-		const void *pData = Client()->SnapGetItem(IClient::SNAP_CURRENT, Index, &Item);
-		const CNetObj_EntityEx *pDataEx = static_cast<CNetObj_EntityEx *>(Client()->SnapFindItem(IClient::SNAP_CURRENT, NETOBJTYPE_ENTITYEX, Item.m_ID));
-		m_GameWorld.NetObjAdd(Item.m_ID, Item.m_Type, pData, pDataEx);
-	}
+	for(const CSnapEntities &EntData : SnapEntities())
+		m_GameWorld.NetObjAdd(EntData.m_Item.m_ID, EntData.m_Item.m_Type, EntData.m_pData, EntData.m_pDataEx);
+
 	m_GameWorld.NetObjEnd(m_Snap.m_LocalClientID);
 
 	// save the characters that are currently active
@@ -3127,3 +3124,43 @@ bool CGameClient::IsDisplayingWarning()
 {
 	return m_Menus.GetCurPopup() == CMenus::POPUP_WARNING;
 }
+
+void CGameClient::SnapCollectEntities()
+{
+	int NumSnapItems = Client()->SnapNumItems(IClient::SNAP_CURRENT);
+
+	std::vector<CSnapEntities> aItemData;
+	for(int Index = 0; Index < NumSnapItems; Index++)
+	{
+		IClient::CSnapItem Item;
+		const void *pData = Client()->SnapGetItem(IClient::SNAP_CURRENT, Index, &Item);
+		if(Item.m_Type == NETOBJTYPE_ENTITYEX || Item.m_Type == NETOBJTYPE_PROJECTILE || Item.m_Type == NETOBJTYPE_PICKUP || Item.m_Type == NETOBJTYPE_LASER || Item.m_Type == NETOBJTYPE_DDNETPROJECTILE)
+			aItemData.push_back({Item, pData, 0});
+	}
+
+	// sort by id, with non-extended items before extended items of the same id
+	std::sort(aItemData.begin(), aItemData.end(), [](const CSnapEntities &lhs, const CSnapEntities &rhs) {
+		if(lhs.m_Item.m_ID == rhs.m_Item.m_ID)
+			return lhs.m_Item.m_Type != NETOBJTYPE_ENTITYEX;
+		return lhs.m_Item.m_ID < rhs.m_Item.m_ID;
+	});
+
+	// merge extended items with items they belong to
+	m_aSnapEntities.clear();
+	for(size_t Index = 0; Index < aItemData.size(); Index++)
+	{
+		if(aItemData[Index].m_Item.m_Type == NETOBJTYPE_ENTITYEX)
+			continue;
+
+		const IClient::CSnapItem Item = aItemData[Index].m_Item;
+		const void *pData = (const void *)aItemData[Index].m_pData;
+		const CNetObj_EntityEx *pDataEx = 0;
+
+		if(Index + 1 < aItemData.size() && aItemData[Index + 1].m_Item.m_ID == Item.m_ID && aItemData[Index + 1].m_Item.m_Type == NETOBJTYPE_ENTITYEX)
+		{
+			pDataEx = (const CNetObj_EntityEx *)aItemData[Index + 1].m_pData;
+			Index++;
+		}
+		m_aSnapEntities.push_back({Item, pData, pDataEx});
+	}
+}

src/game/client/gameclient.h

@@ -92,6 +92,14 @@ public:
 	bool m_AllowXSkins;
 };
 
+class CSnapEntities
+{
+public:
+	IClient::CSnapItem m_Item;
+	const void *m_pData;
+	const CNetObj_EntityEx *m_pDataEx;
+};
+
 class CGameClient : public IGameClient
 {
 public:
@@ -631,7 +639,12 @@ public:
 	SClientEmoticonsSkin m_EmoticonsSkin;
 	bool m_EmoticonsSkinLoaded;
 
+	const std::vector<CSnapEntities> &SnapEntities() { return m_aSnapEntities; }
+
 private:
+	std::vector<CSnapEntities> m_aSnapEntities;
+	void SnapCollectEntities();
+
 	bool m_DDRaceMsgSent[NUM_DUMMIES];
 	int m_ShowOthers[NUM_DUMMIES];
 

src/game/server/entities/door.cpp

@@ -42,6 +42,9 @@ void CDoor::ResetCollision()
 
 void CDoor::Snap(int SnappingClient)
 {
+	if(NetworkClipped(SnappingClient, m_Pos) && NetworkClipped(SnappingClient, m_To))
+		return;
+
 	CNetObj_EntityEx *pEntData = static_cast<CNetObj_EntityEx *>(Server()->SnapNewItem(NETOBJTYPE_ENTITYEX, GetID(), sizeof(CNetObj_EntityEx)));
 	if(!pEntData)
 		return;
@@ -50,9 +53,6 @@ void CDoor::Snap(int SnappingClient)
 	pEntData->m_Layer = m_Layer;
 	pEntData->m_EntityClass = ENTITYCLASS_DOOR;
 
-	if(NetworkClipped(SnappingClient, m_Pos) && NetworkClipped(SnappingClient, m_To))
-		return;
-
 	CNetObj_Laser *pObj = static_cast<CNetObj_Laser *>(Server()->SnapNewItem(
 		NETOBJTYPE_LASER, GetID(), sizeof(CNetObj_Laser)));
 

src/game/server/entities/dragger.cpp

@@ -165,14 +165,6 @@ void CDragger::Snap(int SnappingClient)
 	if(((CGameControllerDDRace *)GameServer()->m_pController)->m_Teams.GetTeamState(m_CaughtTeam) == CGameTeams::TEAMSTATE_EMPTY)
 		return;
 
-	CNetObj_EntityEx *pEntData = static_cast<CNetObj_EntityEx *>(Server()->SnapNewItem(NETOBJTYPE_ENTITYEX, GetID(), sizeof(CNetObj_EntityEx)));
-	if(!pEntData)
-		return;
-
-	pEntData->m_SwitchNumber = m_Number;
-	pEntData->m_Layer = m_Layer;
-	pEntData->m_EntityClass = clamp(ENTITYCLASS_DRAGGER_WEAK + round_to_int(m_Strength) - 1, (int)ENTITYCLASS_DRAGGER_WEAK, (int)ENTITYCLASS_DRAGGER_STRONG);
-
 	int SnappingClientVersion = SnappingClient >= 0 ? GameServer()->GetClientVersion(SnappingClient) : CLIENT_VERSIONNR;
 
 	CCharacter *Target = m_Target;
@@ -241,6 +233,14 @@ void CDragger::Snap(int SnappingClient)
 		{
 			obj = static_cast<CNetObj_Laser *>(Server()->SnapNewItem(
 				NETOBJTYPE_LASER, GetID(), sizeof(CNetObj_Laser)));
+
+			CNetObj_EntityEx *pEntData = static_cast<CNetObj_EntityEx *>(Server()->SnapNewItem(NETOBJTYPE_ENTITYEX, GetID(), sizeof(CNetObj_EntityEx)));
+			if(!pEntData)
+				return;
+
+			pEntData->m_SwitchNumber = m_Number;
+			pEntData->m_Layer = m_Layer;
+			pEntData->m_EntityClass = clamp(ENTITYCLASS_DRAGGER_WEAK + round_to_int(m_Strength) - 1, (int)ENTITYCLASS_DRAGGER_WEAK, (int)ENTITYCLASS_DRAGGER_STRONG);
 		}
 		else
 		{

src/game/server/entities/gun.cpp

@@ -112,6 +112,9 @@ void CGun::Tick()
 
 void CGun::Snap(int SnappingClient)
 {
+	if(NetworkClipped(SnappingClient))
+		return;
+
 	CNetObj_EntityEx *pEntData = static_cast<CNetObj_EntityEx *>(Server()->SnapNewItem(NETOBJTYPE_ENTITYEX, GetID(), sizeof(CNetObj_EntityEx)));
 	if(!pEntData)
 		return;
@@ -128,9 +131,6 @@ void CGun::Snap(int SnappingClient)
 	else
 		pEntData->m_EntityClass = ENTITYCLASS_GUN_UNFREEZE;
 
-	if(NetworkClipped(SnappingClient))
-		return;
-
 	CCharacter *Char = GameServer()->GetPlayerChar(SnappingClient);
 
 	if(SnappingClient > -1 && (GameServer()->m_apPlayers[SnappingClient]->GetTeam() == -1 || GameServer()->m_apPlayers[SnappingClient]->IsPaused()) &&

src/game/server/entities/light.cpp

@@ -102,6 +102,9 @@ void CLight::Tick()
 
 void CLight::Snap(int SnappingClient)
 {
+	if(NetworkClipped(SnappingClient, m_Pos) && NetworkClipped(SnappingClient, m_To))
+		return;
+
 	CNetObj_EntityEx *pEntData = static_cast<CNetObj_EntityEx *>(Server()->SnapNewItem(NETOBJTYPE_ENTITYEX, GetID(), sizeof(CNetObj_EntityEx)));
 	if(!pEntData)
 		return;
@@ -110,9 +113,6 @@ void CLight::Snap(int SnappingClient)
 	pEntData->m_Layer = m_Layer;
 	pEntData->m_EntityClass = ENTITYCLASS_LIGHT;
 
-	if(NetworkClipped(SnappingClient, m_Pos) && NetworkClipped(SnappingClient, m_To))
-		return;
-
 	CCharacter *Char = GameServer()->GetPlayerChar(SnappingClient);
 
 	if(SnappingClient > -1 && (GameServer()->m_apPlayers[SnappingClient]->GetTeam() == -1 || GameServer()->m_apPlayers[SnappingClient]->IsPaused()) && GameServer()->m_apPlayers[SnappingClient]->m_SpectatorID != SPEC_FREEVIEW)

src/game/server/gamecontext.cpp

@@ -3617,10 +3617,6 @@ void CGameContext::OnSnap(int ClientID)
 		Server()->SendMsg(&Msg, MSGFLAG_RECORD | MSGFLAG_NOSEND, ClientID);
 	}
 
-	m_World.Snap(ClientID);
-	m_pController->Snap(ClientID);
-	m_Events.Snap(ClientID);
-
 	for(auto &pPlayer : m_apPlayers)
 	{
 		if(pPlayer)
@@ -3629,6 +3625,10 @@ void CGameContext::OnSnap(int ClientID)
 
 	if(ClientID > -1)
 		m_apPlayers[ClientID]->FakeSnap();
+
+	m_World.Snap(ClientID);
+	m_pController->Snap(ClientID);
+	m_Events.Snap(ClientID);
 }
 void CGameContext::OnPreSnap() {}
 void CGameContext::OnPostSnap()