From 3c3adea4060302bad22fe771d001c07797d37007 Mon Sep 17 00:00:00 2001 From: KingRainbow44 Date: Sat, 9 Sep 2023 20:41:53 -0400 Subject: [PATCH] Add async bulk saving of objects --- .../emu/grasscutter/database/Database.java | 71 +++++++++++++++++++ .../game/activity/PlayerActivityData.java | 25 ++++++- .../emu/grasscutter/game/avatar/Avatar.java | 34 ++++++--- .../game/avatar/AvatarStorage.java | 4 +- .../emu/grasscutter/game/player/Player.java | 41 +++++++---- .../grasscutter/game/player/TeamManager.java | 20 +++--- .../emu/grasscutter/game/world/World.java | 9 +-- .../utils/objects/DatabaseObject.java | 42 +++++++++++ 8 files changed, 209 insertions(+), 37 deletions(-) create mode 100644 src/main/java/emu/grasscutter/database/Database.java create mode 100644 src/main/java/emu/grasscutter/utils/objects/DatabaseObject.java diff --git a/src/main/java/emu/grasscutter/database/Database.java b/src/main/java/emu/grasscutter/database/Database.java new file mode 100644 index 000000000..d9e26eb83 --- /dev/null +++ b/src/main/java/emu/grasscutter/database/Database.java @@ -0,0 +1,71 @@ +package emu.grasscutter.database; + +import emu.grasscutter.Grasscutter; +import emu.grasscutter.utils.objects.DatabaseObject; +import org.slf4j.*; + +import java.util.*; +import java.util.concurrent.*; + +/** + * Complicated manager of the MongoDB database. + * Handles caching, data operations, and more. + */ +public interface Database { + Logger logger = LoggerFactory.getLogger("Database"); + List> objects = new CopyOnWriteArrayList<>(); + + /** + * Queues an object to be saved. + * + * @param object The object to save. + */ + static void save(DatabaseObject object) { + if (object.saveImmediately()) { + object.save(); + } else { + objects.add(object); + } + } + + /** + * Performs a bulk save of all deferred objects. + */ + static void saveAll() { + var size = objects.size(); + Database.saveAll(objects); + + logger.debug("Performed auto save on {} objects.", size); + } + + /** + * Performs a bulk save of all deferred objects. + * + * @param objects The objects to save. + */ + static void saveAll(List> objects) { + // Sort all objects into their respective databases. + var gameObjects = objects.stream() + .filter(DatabaseObject::isGameObject) + .toList(); + var accountObjects = objects.stream() + .filter(o -> !o.isGameObject()) + .toList(); + + // Clear the collective list. + objects.clear(); + + // Save all objects. + var executor = DatabaseHelper.getEventExecutor(); + if (Grasscutter.getRunMode() != Grasscutter.ServerRunMode.DISPATCH_ONLY) { + executor.submit(() -> { + DatabaseManager.getGameDatastore().save(gameObjects); + }); + } + if (Grasscutter.getRunMode() != Grasscutter.ServerRunMode.GAME_ONLY) { + executor.submit(() -> { + DatabaseManager.getAccountDatastore().save(accountObjects); + }); + } + } +} diff --git a/src/main/java/emu/grasscutter/game/activity/PlayerActivityData.java b/src/main/java/emu/grasscutter/game/activity/PlayerActivityData.java index 61ecf9d7c..254836184 100644 --- a/src/main/java/emu/grasscutter/game/activity/PlayerActivityData.java +++ b/src/main/java/emu/grasscutter/game/activity/PlayerActivityData.java @@ -11,15 +11,17 @@ import emu.grasscutter.game.props.ActionReason; import emu.grasscutter.net.proto.ActivityWatcherInfoOuterClass; import emu.grasscutter.server.packet.send.PacketActivityUpdateWatcherNotify; import emu.grasscutter.utils.JsonUtils; -import java.util.*; +import emu.grasscutter.utils.objects.DatabaseObject; import lombok.*; import lombok.experimental.FieldDefaults; +import java.util.*; + @Entity("activities") @Data @FieldDefaults(level = AccessLevel.PRIVATE) @Builder(builderMethodName = "of") -public class PlayerActivityData { +public class PlayerActivityData implements DatabaseObject { @Id String id; int uid; int activityId; @@ -34,8 +36,25 @@ public class PlayerActivityData { return DatabaseHelper.getPlayerActivityData(player.getUid(), activityId); } + /** + * Saves this object to the database. + * As of Grasscutter 1.7.1, this is by default a {@link DatabaseObject#deferSave()} call. + */ public void save() { - DatabaseHelper.savePlayerActivityData(this); + this.deferSave(); + } + + /** + * Saves this object to the database. + * + * @param immediate If true, this will be a {@link DatabaseObject#save()} call instead of a {@link DatabaseObject#deferSave()} call. + */ + public void save(boolean immediate) { + if (immediate) { + DatabaseObject.super.save(); + } else { + this.save(); + } } public synchronized void addWatcherProgress(int watcherId) { diff --git a/src/main/java/emu/grasscutter/game/avatar/Avatar.java b/src/main/java/emu/grasscutter/game/avatar/Avatar.java index be5774ed0..3bf4b2c2c 100644 --- a/src/main/java/emu/grasscutter/game/avatar/Avatar.java +++ b/src/main/java/emu/grasscutter/game/avatar/Avatar.java @@ -1,7 +1,5 @@ package emu.grasscutter.game.avatar; -import static emu.grasscutter.config.Configuration.GAME_OPTIONS; - import dev.morphia.annotations.*; import emu.grasscutter.GameConstants; import emu.grasscutter.data.GameData; @@ -15,7 +13,6 @@ import emu.grasscutter.data.excels.avatar.AvatarSkillDepotData.InherentProudSkil import emu.grasscutter.data.excels.reliquary.*; import emu.grasscutter.data.excels.trial.TrialAvatarTemplateData; import emu.grasscutter.data.excels.weapon.*; -import emu.grasscutter.database.DatabaseHelper; import emu.grasscutter.game.entity.*; import emu.grasscutter.game.inventory.*; import emu.grasscutter.game.player.Player; @@ -31,15 +28,19 @@ import emu.grasscutter.net.proto.TrialAvatarGrantRecordOuterClass.TrialAvatarGra import emu.grasscutter.net.proto.TrialAvatarInfoOuterClass.TrialAvatarInfo; import emu.grasscutter.server.packet.send.*; import emu.grasscutter.utils.helpers.ProtoHelper; +import emu.grasscutter.utils.objects.DatabaseObject; import it.unimi.dsi.fastutil.ints.*; -import java.util.*; -import java.util.stream.Stream; -import javax.annotation.*; import lombok.*; import org.bson.types.ObjectId; +import javax.annotation.*; +import java.util.*; +import java.util.stream.Stream; + +import static emu.grasscutter.config.Configuration.GAME_OPTIONS; + @Entity(value = "avatars", useDiscriminator = false) -public class Avatar { +public class Avatar implements DatabaseObject { @Transient @Getter private final Int2ObjectMap equips; @Transient @Getter private final Int2FloatOpenHashMap fightProperties; @Transient @Getter private final Int2FloatOpenHashMap fightPropOverrides; @@ -989,8 +990,25 @@ public class Avatar { return entity != null ? entity.getId() : 0; } + /** + * Saves this object to the database. + * As of Grasscutter 1.7.1, this is by default a {@link DatabaseObject#deferSave()} call. + */ public void save() { - DatabaseHelper.saveAvatar(this); + this.deferSave(); + } + + /** + * Saves this object to the database. + * + * @param immediate If true, this will be a {@link DatabaseObject#save()} call instead of a {@link DatabaseObject#deferSave()} call. + */ + public void save(boolean immediate) { + if (immediate) { + DatabaseObject.super.save(); + } else { + this.save(); + } } public AvatarInfo toProto() { diff --git a/src/main/java/emu/grasscutter/game/avatar/AvatarStorage.java b/src/main/java/emu/grasscutter/game/avatar/AvatarStorage.java index 1d2002e7b..af4617179 100644 --- a/src/main/java/emu/grasscutter/game/avatar/AvatarStorage.java +++ b/src/main/java/emu/grasscutter/game/avatar/AvatarStorage.java @@ -60,7 +60,7 @@ public class AvatarStorage extends BasePlayerManager implements Iterable this.avatars.put(avatar.getAvatarId(), avatar); this.avatarsGuid.put(avatar.getGuid(), avatar); - avatar.save(); + avatar.save(true); return true; } @@ -165,7 +165,7 @@ public class AvatarStorage extends BasePlayerManager implements Iterable if ((avatar.getAvatarId() == 10000007) || (avatar.getAvatarId() == 10000005)) { avatar.setSkillDepot(skillDepot); avatar.setSkillDepotData(skillDepot); - avatar.save(); + avatar.save(true); } } diff --git a/src/main/java/emu/grasscutter/game/player/Player.java b/src/main/java/emu/grasscutter/game/player/Player.java index ac174e5b2..912c2b63c 100644 --- a/src/main/java/emu/grasscutter/game/player/Player.java +++ b/src/main/java/emu/grasscutter/game/player/Player.java @@ -55,7 +55,7 @@ import emu.grasscutter.server.game.GameSession.SessionState; import emu.grasscutter.server.packet.send.*; import emu.grasscutter.utils.*; import emu.grasscutter.utils.helpers.DateHelper; -import emu.grasscutter.utils.objects.FieldFetch; +import emu.grasscutter.utils.objects.*; import it.unimi.dsi.fastutil.ints.*; import lombok.*; @@ -66,7 +66,7 @@ import java.util.concurrent.*; import static emu.grasscutter.config.Configuration.GAME_OPTIONS; @Entity(value = "players", useDiscriminator = false) -public class Player implements PlayerHook, FieldFetch { +public class Player implements DatabaseObject, PlayerHook, FieldFetch { @Id private int id; @Indexed(options = @IndexOptions(unique = true)) @Getter private String accountId; @@ -1306,8 +1306,25 @@ public class Player implements PlayerHook, FieldFetch { this.getTeamManager().setPlayer(this); } + /** + * Saves this object to the database. + * As of Grasscutter 1.7.1, this is by default a {@link DatabaseObject#deferSave()} call. + */ public void save() { - DatabaseHelper.savePlayer(this); + this.deferSave(); + } + + /** + * Saves this object to the database. + * + * @param immediate If true, this will be a {@link DatabaseObject#save()} call instead of a {@link DatabaseObject#deferSave()} call. + */ + public void save(boolean immediate) { + if (immediate) { + DatabaseObject.super.save(); + } else { + this.save(); + } } // Called from tokenrsp @@ -1474,20 +1491,19 @@ public class Player implements PlayerHook, FieldFetch { this.getProfile().syncWithCharacter(this); this.getCoopRequests().clear(); - this.getEnterHomeRequests().values().forEach(req -> this.expireEnterHomeRequest(req, true)); + this.getEnterHomeRequests().values() + .forEach(req -> this.expireEnterHomeRequest(req, true)); this.getEnterHomeRequests().clear(); // Save to db - this.save(); + this.save(true); this.getTeamManager().saveAvatars(); this.getFriendsList().save(); // Call quit event. - PlayerQuitEvent event = new PlayerQuitEvent(this); - event.call(); + new PlayerQuitEvent(this).call(); } catch (Throwable e) { - e.printStackTrace(); - Grasscutter.getLogger().warn("Player (UID {}) save failure", getUid()); + Grasscutter.getLogger().warn("Player (UID {}) failed to save.", this.getUid(), e); } finally { removeFromServer(); } @@ -1495,9 +1511,10 @@ public class Player implements PlayerHook, FieldFetch { public void removeFromServer() { // Remove from server. - //Note: DON'T DELETE BY UID,BECAUSE THERE ARE MULTIPLE SAME UID PLAYERS WHEN DUPLICATED LOGIN! - //so I decide to delete by object rather than uid - getServer().getPlayers().values().removeIf(player1 -> player1 == this); + // Note: DON'T DELETE BY UID, BECAUSE THERE ARE MULTIPLE SAME UID PLAYERS WHEN DUPLICATED LOGIN! + //s o I decide to delete by object rather than uid + this.getServer().getPlayers().values() + .removeIf(player1 -> player1 == this); } public int getLegendaryKey() { diff --git a/src/main/java/emu/grasscutter/game/player/TeamManager.java b/src/main/java/emu/grasscutter/game/player/TeamManager.java index f6a038e31..a51d58ace 100644 --- a/src/main/java/emu/grasscutter/game/player/TeamManager.java +++ b/src/main/java/emu/grasscutter/game/player/TeamManager.java @@ -1,11 +1,10 @@ package emu.grasscutter.game.player; -import static emu.grasscutter.config.Configuration.GAME_OPTIONS; - import dev.morphia.annotations.*; import emu.grasscutter.*; import emu.grasscutter.data.GameData; import emu.grasscutter.data.excels.avatar.AvatarSkillDepotData; +import emu.grasscutter.database.Database; import emu.grasscutter.game.avatar.Avatar; import emu.grasscutter.game.entity.*; import emu.grasscutter.game.props.*; @@ -23,9 +22,12 @@ import emu.grasscutter.server.packet.send.*; import emu.grasscutter.utils.Utils; import it.unimi.dsi.fastutil.ints.*; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import lombok.*; + import java.util.*; import java.util.stream.Stream; -import lombok.*; + +import static emu.grasscutter.config.Configuration.GAME_OPTIONS; @Entity public final class TeamManager extends BasePlayerDataManager { @@ -404,7 +406,7 @@ public final class TeamManager extends BasePlayerDataManager { // Unload removed entities for (var entity : existingAvatars.values()) { this.getPlayer().getScene().removeEntity(entity); - entity.getAvatar().save(); + entity.getAvatar().save(true); } // Set new selected character index @@ -965,11 +967,13 @@ public final class TeamManager extends BasePlayerDataManager { return respawnPoint.get().getPointData().getTranPos(); } + /** + * Performs a bulk save operation on all avatars. + */ public void saveAvatars() { - // Save all avatars from active team - for (EntityAvatar entity : this.getActiveTeam()) { - entity.getAvatar().save(); - } + Database.saveAll(this.getActiveTeam().stream() + .map(EntityAvatar::getAvatar) + .toList()); } public void onPlayerLogin() { // Hack for now to fix resonances on login diff --git a/src/main/java/emu/grasscutter/game/world/World.java b/src/main/java/emu/grasscutter/game/world/World.java index 3b9e01ed0..8f507d7c4 100644 --- a/src/main/java/emu/grasscutter/game/world/World.java +++ b/src/main/java/emu/grasscutter/game/world/World.java @@ -1,7 +1,5 @@ package emu.grasscutter.game.world; -import static emu.grasscutter.server.event.player.PlayerTeleportEvent.TeleportType.SCRIPT; - import emu.grasscutter.data.GameData; import emu.grasscutter.data.excels.dungeon.DungeonData; import emu.grasscutter.game.entity.*; @@ -20,10 +18,13 @@ import emu.grasscutter.server.game.GameServer; import emu.grasscutter.server.packet.send.*; import emu.grasscutter.utils.ConversionUtils; import it.unimi.dsi.fastutil.ints.*; -import java.util.*; import lombok.*; import org.jetbrains.annotations.NotNull; +import java.util.*; + +import static emu.grasscutter.server.event.player.PlayerTeleportEvent.TeleportType.SCRIPT; + public class World implements Iterable { @Getter private final GameServer server; @Getter private Player host; @@ -266,7 +267,7 @@ public class World implements Iterable { scene.removePlayer(player); // Info packet for other players - if (this.getPlayers().size() > 0) { + if (!this.getPlayers().isEmpty()) { this.updatePlayerInfos(player); } diff --git a/src/main/java/emu/grasscutter/utils/objects/DatabaseObject.java b/src/main/java/emu/grasscutter/utils/objects/DatabaseObject.java new file mode 100644 index 000000000..8478a5370 --- /dev/null +++ b/src/main/java/emu/grasscutter/utils/objects/DatabaseObject.java @@ -0,0 +1,42 @@ +package emu.grasscutter.utils.objects; + +import emu.grasscutter.Grasscutter; +import emu.grasscutter.Grasscutter.ServerRunMode; +import emu.grasscutter.database.*; + +public interface DatabaseObject { + /** + * @return Does this object belong in the game database? + */ + default boolean isGameObject() { + return true; + } + + /** + * @return Should this object be saved immediately? + */ + default boolean saveImmediately() { + return false; + } + + /** + * Performs a deferred save. + * This object will save as a group with other objects. + */ + default void deferSave() { + Database.save(this); + } + + /** + * Attempts to save this object to the database. + */ + default void save() { + if (this.isGameObject()) { + DatabaseManager.getGameDatastore().save(this); + } else if (Grasscutter.getRunMode() != ServerRunMode.GAME_ONLY) { + DatabaseManager.getAccountDatastore().save(this); + } else { + throw new UnsupportedOperationException("Unable to store an account object while in game-only mode."); + } + } +}