From 0994417261d93bc6bb7e9f74d55ae1a06701c60b Mon Sep 17 00:00:00 2001 From: AnimeGitB Date: Sat, 30 Jul 2022 20:52:59 +0930 Subject: [PATCH] Cache used strings from TextMaps --- .gitignore | 1 + .../emu/grasscutter/data/ResourceLoader.java | 3 + .../java/emu/grasscutter/tools/Tools.java | 113 +--------- .../java/emu/grasscutter/utils/Language.java | 212 ++++++++++++++++++ 4 files changed, 222 insertions(+), 107 deletions(-) diff --git a/.gitignore b/.gitignore index 234d23af7..1a1f39e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ tmp/ .vscode # Grasscutter +/cache /resources /logs /plugins diff --git a/src/main/java/emu/grasscutter/data/ResourceLoader.java b/src/main/java/emu/grasscutter/data/ResourceLoader.java index f618b4969..01662e23f 100644 --- a/src/main/java/emu/grasscutter/data/ResourceLoader.java +++ b/src/main/java/emu/grasscutter/data/ResourceLoader.java @@ -53,7 +53,9 @@ public class ResourceLoader { return classList; } + private static boolean loadedAll = false; public static void loadAll() { + if (loadedAll) return; Grasscutter.getLogger().info(translate("messages.status.resources.loading")); // Load ability lists @@ -75,6 +77,7 @@ public class ResourceLoader { loadNpcBornData(); Grasscutter.getLogger().info(translate("messages.status.resources.finish")); + loadedAll = true; } public static void loadResources() { diff --git a/src/main/java/emu/grasscutter/tools/Tools.java b/src/main/java/emu/grasscutter/tools/Tools.java index 6be272c27..e7be64b61 100644 --- a/src/main/java/emu/grasscutter/tools/Tools.java +++ b/src/main/java/emu/grasscutter/tools/Tools.java @@ -1,10 +1,8 @@ package emu.grasscutter.tools; -import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; -import java.io.FileReader; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.PrintWriter; @@ -12,10 +10,7 @@ import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.IntStream; import com.google.gson.reflect.TypeToken; @@ -33,119 +28,25 @@ import emu.grasscutter.data.excels.QuestData; import emu.grasscutter.data.excels.SceneData; import emu.grasscutter.utils.Language; import emu.grasscutter.utils.Utils; +import emu.grasscutter.utils.Language.TextStrings; import it.unimi.dsi.fastutil.ints.Int2IntMap; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; -import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.ints.IntArrayList; -import it.unimi.dsi.fastutil.ints.IntList; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.ints.IntSet; -import it.unimi.dsi.fastutil.objects.Object2IntMap; -import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; -import lombok.EqualsAndHashCode; import static emu.grasscutter.config.Configuration.*; -import static emu.grasscutter.utils.Language.translate; public final class Tools { - @EqualsAndHashCode public static class TextStrings { - public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"}; - public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"}; - public static final int NUM_LANGUAGES = ARR_LANGUAGES.length; - public static final List LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES); - public static final Object2IntMap MAP_LANGUAGES = // Map "EN": 0, "CHS": 1, ..., "VI": 12 - new Object2IntOpenHashMap<>( - IntStream.range(0, ARR_LANGUAGES.length) - .boxed() - .collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i))); - public String[] strings = new String[ARR_LANGUAGES.length]; - - public TextStrings() {}; - - public TextStrings(String init) { - for (int i = 0; i < NUM_LANGUAGES; i++) - this.strings[i] = init; - }; - - public TextStrings(Collection strings) { - this.strings = strings.toArray(new String[0]); - } - - public String get(String languageCode) { - return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)]; - } - - public boolean set(String languageCode, String string) { - int index = MAP_LANGUAGES.getOrDefault(languageCode, -1); - if (index < 0) return false; - strings[index] = string; - return true; - } - } - - private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\""); - - private static Int2ObjectMap loadTextMap(String language, IntSet nameHashes) { - Int2ObjectMap output = new Int2ObjectOpenHashMap<>(); - try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) { - Matcher matcher = textMapKeyValueRegex.matcher(""); - return new Int2ObjectOpenHashMap<>( - file.lines() - .sequential() - .map(matcher::reset) // Side effects, but it's faster than making a new one - .filter(Matcher::find) - .filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1)))) // TODO: Cache this parse somehow - .collect(Collectors.toMap( - m -> (int) Long.parseLong(m.group(1)), - m -> m.group(2)))); - } catch (Exception e) { - Grasscutter.getLogger().error("Error loading textmap: " + language); - Grasscutter.getLogger().error(e.toString()); - } - return output; - } - - public static Int2ObjectMap loadTextMaps(IntSet nameHashes) { - Map> mapLanguageMaps = // Separate step to process the textmaps in parallel - TextStrings.LIST_LANGUAGES.parallelStream().collect( - Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMap(s, nameHashes))); - List> languageMaps = - IntStream.range(0, TextStrings.NUM_LANGUAGES) - .mapToObj(i -> mapLanguageMaps.get(i)) - .collect(Collectors.toList()); - - Map canonicalTextStrings = new HashMap<>(); - return new Int2ObjectOpenHashMap( - nameHashes - .intStream() - .boxed() - .collect(Collectors.toMap(key -> key, key -> { - TextStrings t = new TextStrings( - IntStream.range(0, TextStrings.NUM_LANGUAGES) - .mapToObj(i -> languageMaps.get(i).getOrDefault((int) key, "[N/A] - hash key %d".formatted(key))) - .collect(Collectors.toList())); - return canonicalTextStrings.computeIfAbsent(t, x -> t); - })) - ); - } - public static void createGmHandbooks() throws Exception { ResourceLoader.loadAll(); Int2IntMap avatarNames = new Int2IntOpenHashMap(GameData.getAvatarDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash()))); Int2IntMap itemNames = new Int2IntOpenHashMap(GameData.getItemDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash()))); Int2IntMap monsterNames = new Int2IntOpenHashMap(GameData.getMonsterDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash()))); Int2IntMap mainQuestTitles = new Int2IntOpenHashMap(GameData.getMainQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getTitleTextMapHash()))); - Int2IntMap questDescs = new Int2IntOpenHashMap(GameData.getQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getDescTextMapHash()))); - - IntSet usedHashes = new IntOpenHashSet(); - usedHashes.addAll(avatarNames.values()); - usedHashes.addAll(itemNames.values()); - usedHashes.addAll(monsterNames.values()); - usedHashes.addAll(mainQuestTitles.values()); - usedHashes.addAll(questDescs.values()); - - Int2ObjectMap textMaps = loadTextMaps(usedHashes); + // Int2IntMap questDescs = new Int2IntOpenHashMap(GameData.getQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getDescTextMapHash()))); + + Int2ObjectMap textMaps = Language.getTextMapStrings(); Language savedLanguage = Grasscutter.getLanguage(); @@ -233,10 +134,8 @@ public final class Tools { var questDataMap = GameData.getQuestDataMap(); questDataMap.keySet().intStream().sorted().forEach(id -> { QuestData data = questDataMap.get(id); - int titleKey = (int) mainQuestTitles.get(data.getMainId()); - int descKey = (int) data.getDescTextMapHash(); - TextStrings title = textMaps.get(titleKey); - TextStrings desc = textMaps.get(descKey); + TextStrings title = textMaps.get((int) mainQuestTitles.get(data.getMainId())); + TextStrings desc = textMaps.get((int) data.getDescTextMapHash()); for (int i = 0; i < TextStrings.NUM_LANGUAGES; i++) { handbookBuilders[i] .append(id) diff --git a/src/main/java/emu/grasscutter/utils/Language.java b/src/main/java/emu/grasscutter/utils/Language.java index 627c0f310..3060906da 100644 --- a/src/main/java/emu/grasscutter/utils/Language.java +++ b/src/main/java/emu/grasscutter/utils/Language.java @@ -3,14 +3,43 @@ package emu.grasscutter.utils; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import emu.grasscutter.Grasscutter; +import emu.grasscutter.data.GameData; +import emu.grasscutter.data.ResourceLoader; import emu.grasscutter.game.player.Player; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; +import it.unimi.dsi.fastutil.ints.IntSet; +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import lombok.EqualsAndHashCode; import javax.annotation.Nullable; import static emu.grasscutter.config.Configuration.*; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import java.util.Map; public final class Language { @@ -207,4 +236,187 @@ public final class Language { return languageFile; } } + + private static final int TEXTMAP_CACHE_VERSION = 0x9CCACE02; + @EqualsAndHashCode public static class TextStrings implements Serializable { + public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"}; + public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"}; + public static final int NUM_LANGUAGES = ARR_LANGUAGES.length; + public static final List LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES); + public static final Object2IntMap MAP_LANGUAGES = // Map "EN": 0, "CHS": 1, ..., "VI": 12 + new Object2IntOpenHashMap<>( + IntStream.range(0, ARR_LANGUAGES.length) + .boxed() + .collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i))); + public String[] strings = new String[ARR_LANGUAGES.length]; + + public TextStrings() {}; + + public TextStrings(String init) { + for (int i = 0; i < NUM_LANGUAGES; i++) + this.strings[i] = init; + }; + + public TextStrings(List strings, int key) { + // Some hashes don't have strings for some languages :( + String nullReplacement = "[N/A] %d".formatted((long) key & 0xFFFFFFFFL); + for (int i = 0; i < NUM_LANGUAGES; i++) { // Find first non-null if there is any + String s = strings.get(i); + if (s != null) { + nullReplacement = "[%s] - %s".formatted(ARR_LANGUAGES[i], s); + break; + } + } + for (int i = 0; i < NUM_LANGUAGES; i++) { + String s = strings.get(i); + if (s != null) + this.strings[i] = s; + else + this.strings[i] = nullReplacement; + } + } + + public String get(String languageCode) { + return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)]; + } + + public boolean set(String languageCode, String string) { + int index = MAP_LANGUAGES.getOrDefault(languageCode, -1); + if (index < 0) return false; + strings[index] = string; + return true; + } + } + + private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\""); + + private static Int2ObjectMap loadTextMapFile(String language, IntSet nameHashes) { + Int2ObjectMap output = new Int2ObjectOpenHashMap<>(); + try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) { + Matcher matcher = textMapKeyValueRegex.matcher(""); + return new Int2ObjectOpenHashMap<>( + file.lines() + .sequential() + .map(matcher::reset) // Side effects, but it's faster than making a new one + .filter(Matcher::find) + .filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1)))) // TODO: Cache this parse somehow + .collect(Collectors.toMap( + m -> (int) Long.parseLong(m.group(1)), + m -> m.group(2).replace("\\\"", "\"")))); + } catch (Exception e) { + Grasscutter.getLogger().error("Error loading textmap: " + language); + Grasscutter.getLogger().error(e.toString()); + } + return output; + } + + private static Int2ObjectMap loadTextMapFiles(IntSet nameHashes) { + Map> mapLanguageMaps = // Separate step to process the textmaps in parallel + TextStrings.LIST_LANGUAGES.parallelStream().collect( + Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMapFile(s, nameHashes))); + List> languageMaps = + IntStream.range(0, TextStrings.NUM_LANGUAGES) + .mapToObj(i -> mapLanguageMaps.get(i)) + .collect(Collectors.toList()); + + Map canonicalTextStrings = new HashMap<>(); + return new Int2ObjectOpenHashMap( + nameHashes + .intStream() + .boxed() + .collect(Collectors.toMap(key -> key, key -> { + TextStrings t = new TextStrings( + IntStream.range(0, TextStrings.NUM_LANGUAGES) + .mapToObj(i -> languageMaps.get(i).get((int) key)) + .collect(Collectors.toList()), (int) key); + return canonicalTextStrings.computeIfAbsent(t, x -> t); + })) + ); + } + + private static Int2ObjectMap loadTextMapsCache() throws Exception { + try (ObjectInputStream file = new ObjectInputStream(new BufferedInputStream(Files.newInputStream(TEXTMAP_CACHE_PATH), 0x100000))) { + final int fileVersion = file.readInt(); + if (fileVersion != TEXTMAP_CACHE_VERSION) + throw new Exception("Invalid cache version"); + return (Int2ObjectMap) file.readObject(); + } + } + + private static void saveTextMapsCache(Int2ObjectMap input) throws IOException { + try { + Files.createDirectory(Path.of("cache")); + } catch (FileAlreadyExistsException ignored) {}; + try (ObjectOutputStream file = new ObjectOutputStream(new BufferedOutputStream(Files.newOutputStream(TEXTMAP_CACHE_PATH, StandardOpenOption.CREATE), 0x100000))) { + file.writeInt(TEXTMAP_CACHE_VERSION); + file.writeObject(input); + } + } + + private static Int2ObjectMap textMapStrings; + private static final Path TEXTMAP_CACHE_PATH = Path.of(Utils.toFilePath("cache/TextMapCache.bin")); + + public static Int2ObjectMap getTextMapStrings() { + if (textMapStrings == null) + loadTextMaps(); + return textMapStrings; + } + + public static TextStrings getTextMapKey(long hash) { + return textMapStrings.get((int) hash); + } + + public static void loadTextMaps() { + // Check system timestamps on cache and resources + try { + long cacheModified = Files.getLastModifiedTime(TEXTMAP_CACHE_PATH).toMillis(); + + long textmapsModified = Files.list(Path.of(RESOURCE("TextMap"))) + .filter(path -> path.toString().endsWith(".json")) + .map(path -> { + try { + return Files.getLastModifiedTime(path).toMillis(); + } catch (Exception ignored) { + Grasscutter.getLogger().debug("Exception while checking modified time: ", path); + return Long.MAX_VALUE; // Don't use cache, something has gone wrong + } + }) + .max(Long::compare) + .get(); + + Grasscutter.getLogger().debug("Cache modified %d, textmap modified %d".formatted(cacheModified, textmapsModified)); + if (textmapsModified < cacheModified) { + // Try loading from cache + Grasscutter.getLogger().info("Loading cached TextMaps"); + textMapStrings = loadTextMapsCache(); + return; + } + } catch (Exception e) { + Grasscutter.getLogger().debug("Exception while checking cache: ", e); + }; + + // Regenerate cache + Grasscutter.getLogger().info("Generating TextMaps cache"); + ResourceLoader.loadAll(); + IntSet usedHashes = new IntOpenHashSet(); + GameData.getAvatarDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash())); + GameData.getItemDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash())); + GameData.getMonsterDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash())); + GameData.getMainQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getTitleTextMapHash())); + GameData.getQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getDescTextMapHash())); + // Incidental strings + usedHashes.add((int) 4233146695L); // Character + usedHashes.add((int) 4231343903L); // Weapon + usedHashes.add((int) 332935371L); // Standard Wish + usedHashes.add((int) 2272170627L); // Character Event Wish + usedHashes.add((int) 3352513147L); // Character Event Wish-2 + usedHashes.add((int) 2864268523L); // Weapon Event Wish + + textMapStrings = loadTextMapFiles(usedHashes); + try { + saveTextMapsCache(textMapStrings); + } catch (IOException e) { + Grasscutter.getLogger().error("Failed to save TextMap cache: ", e); + }; + } }