Cache used strings from TextMaps

.gitignore

@@ -52,6 +52,7 @@ tmp/
 .vscode
 
 # Grasscutter
+/cache
 /resources
 /logs
 /plugins

src/main/java/emu/grasscutter/data/ResourceLoader.java

@@ -53,7 +53,9 @@ public class ResourceLoader {
         return classList;
     }
 
+    private static boolean loadedAll = false;
     public static void loadAll() {
+        if (loadedAll) return;
         Grasscutter.getLogger().info(translate("messages.status.resources.loading"));
 
 		// Load ability lists
@@ -75,6 +77,7 @@ public class ResourceLoader {
 		loadNpcBornData();
 
         Grasscutter.getLogger().info(translate("messages.status.resources.finish"));
+        loadedAll = true;
     }
 
     public static void loadResources() {

src/main/java/emu/grasscutter/tools/Tools.java

@@ -1,10 +1,8 @@
 package emu.grasscutter.tools;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.io.FileReader;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
@@ -12,10 +10,7 @@ import java.nio.charset.StandardCharsets;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
 import com.google.gson.reflect.TypeToken;
 
@@ -33,119 +28,25 @@ import emu.grasscutter.data.excels.QuestData;
 import emu.grasscutter.data.excels.SceneData;
 import emu.grasscutter.utils.Language;
 import emu.grasscutter.utils.Utils;
+import emu.grasscutter.utils.Language.TextStrings;
 import it.unimi.dsi.fastutil.ints.Int2IntMap;
 import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
-import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
-import it.unimi.dsi.fastutil.ints.IntArrayList;
-import it.unimi.dsi.fastutil.ints.IntList;
 import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
 import it.unimi.dsi.fastutil.ints.IntSet;
-import it.unimi.dsi.fastutil.objects.Object2IntMap;
-import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
-import lombok.EqualsAndHashCode;
 
 import static emu.grasscutter.config.Configuration.*;
-import static emu.grasscutter.utils.Language.translate;
 
 public final class Tools {
-    @EqualsAndHashCode public static class TextStrings {
-        public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"};
-        public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"};
-        public static final int NUM_LANGUAGES = ARR_LANGUAGES.length;
-        public static final List<String> LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES);
-        public static final Object2IntMap<String> MAP_LANGUAGES =  // Map "EN": 0, "CHS": 1, ..., "VI": 12
-            new Object2IntOpenHashMap<>(
-                IntStream.range(0, ARR_LANGUAGES.length)
-                .boxed()
-                .collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i)));
-        public String[] strings = new String[ARR_LANGUAGES.length];
-
-        public TextStrings() {};
-
-        public TextStrings(String init) {
-            for (int i = 0; i < NUM_LANGUAGES; i++)
-                this.strings[i] = init;
-        };
-
-        public TextStrings(Collection<String> strings) {
-            this.strings = strings.toArray(new String[0]);
-        }
-
-        public String get(String languageCode) {
-            return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)];
-        }
-
-        public boolean set(String languageCode, String string) {
-            int index = MAP_LANGUAGES.getOrDefault(languageCode, -1);
-            if (index < 0) return false;
-            strings[index] = string;
-            return true;
-        }
-    }
-
-    private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\"");
-
-    private static Int2ObjectMap<String> loadTextMap(String language, IntSet nameHashes) {
-        Int2ObjectMap<String> output = new Int2ObjectOpenHashMap<>();
-        try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) {
-            Matcher matcher = textMapKeyValueRegex.matcher("");
-            return new Int2ObjectOpenHashMap<>(
-                file.lines()
-                    .sequential()
-                    .map(matcher::reset)  // Side effects, but it's faster than making a new one
-                    .filter(Matcher::find)
-                    .filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1))))  // TODO: Cache this parse somehow
-                    .collect(Collectors.toMap(
-                        m -> (int) Long.parseLong(m.group(1)),
-                        m -> m.group(2))));
-        } catch (Exception e) {
-            Grasscutter.getLogger().error("Error loading textmap: " + language);
-            Grasscutter.getLogger().error(e.toString());
-        }
-        return output;
-    }
-
-    public static Int2ObjectMap<TextStrings> loadTextMaps(IntSet nameHashes) {
-        Map<Integer, Int2ObjectMap<String>> mapLanguageMaps =  // Separate step to process the textmaps in parallel
-            TextStrings.LIST_LANGUAGES.parallelStream().collect(
-            Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMap(s, nameHashes)));
-        List<Int2ObjectMap<String>> languageMaps = 
-            IntStream.range(0, TextStrings.NUM_LANGUAGES)
-            .mapToObj(i -> mapLanguageMaps.get(i))
-            .collect(Collectors.toList());
-
-        Map<TextStrings, TextStrings> canonicalTextStrings = new HashMap<>();
-        return new Int2ObjectOpenHashMap<TextStrings>(
-            nameHashes
-            .intStream()
-            .boxed()
-            .collect(Collectors.toMap(key -> key, key -> {
-                TextStrings t = new TextStrings(
-                    IntStream.range(0, TextStrings.NUM_LANGUAGES)
-                    .mapToObj(i -> languageMaps.get(i).getOrDefault((int) key, "[N/A] - hash key %d".formatted(key)))
-                    .collect(Collectors.toList()));
-                return canonicalTextStrings.computeIfAbsent(t, x -> t);
-                }))
-            );
-    }
-
     public static void createGmHandbooks() throws Exception {
         ResourceLoader.loadAll();
         Int2IntMap avatarNames = new Int2IntOpenHashMap(GameData.getAvatarDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
         Int2IntMap itemNames = new Int2IntOpenHashMap(GameData.getItemDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
         Int2IntMap monsterNames = new Int2IntOpenHashMap(GameData.getMonsterDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
         Int2IntMap mainQuestTitles = new Int2IntOpenHashMap(GameData.getMainQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getTitleTextMapHash())));
-        Int2IntMap questDescs = new Int2IntOpenHashMap(GameData.getQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getDescTextMapHash())));
+        // Int2IntMap questDescs = new Int2IntOpenHashMap(GameData.getQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getDescTextMapHash())));
-        
+
-        IntSet usedHashes = new IntOpenHashSet();
+        Int2ObjectMap<TextStrings> textMaps = Language.getTextMapStrings();
-        usedHashes.addAll(avatarNames.values());
-        usedHashes.addAll(itemNames.values());
-        usedHashes.addAll(monsterNames.values());
-        usedHashes.addAll(mainQuestTitles.values());
-        usedHashes.addAll(questDescs.values());
-        
-        Int2ObjectMap<TextStrings> textMaps = loadTextMaps(usedHashes);
 
         Language savedLanguage = Grasscutter.getLanguage();
 
@@ -233,10 +134,8 @@ public final class Tools {
         var questDataMap = GameData.getQuestDataMap();
         questDataMap.keySet().intStream().sorted().forEach(id -> {
             QuestData data = questDataMap.get(id);
-            int titleKey = (int) mainQuestTitles.get(data.getMainId());
+            TextStrings title = textMaps.get((int) mainQuestTitles.get(data.getMainId()));
-            int descKey = (int) data.getDescTextMapHash();
+            TextStrings desc = textMaps.get((int) data.getDescTextMapHash());
-            TextStrings title = textMaps.get(titleKey);
-            TextStrings desc = textMaps.get(descKey);
             for (int i = 0; i < TextStrings.NUM_LANGUAGES; i++) {
                 handbookBuilders[i]
                     .append(id)

src/main/java/emu/grasscutter/utils/Language.java

@@ -3,14 +3,43 @@ package emu.grasscutter.utils;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import emu.grasscutter.Grasscutter;
+import emu.grasscutter.data.GameData;
+import emu.grasscutter.data.ResourceLoader;
 import emu.grasscutter.game.player.Player;
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
+import it.unimi.dsi.fastutil.ints.IntSet;
+import it.unimi.dsi.fastutil.objects.Object2IntMap;
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import lombok.EqualsAndHashCode;
 
 import javax.annotation.Nullable;
 
 import static emu.grasscutter.config.Configuration.*;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
 import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public final class Language {
@@ -207,4 +236,187 @@ public final class Language {
             return languageFile;
         }
     }
+
+    private static final int TEXTMAP_CACHE_VERSION = 0x9CCACE02;
+    @EqualsAndHashCode public static class TextStrings implements Serializable {
+        public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"};
+        public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"};
+        public static final int NUM_LANGUAGES = ARR_LANGUAGES.length;
+        public static final List<String> LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES);
+        public static final Object2IntMap<String> MAP_LANGUAGES =  // Map "EN": 0, "CHS": 1, ..., "VI": 12
+            new Object2IntOpenHashMap<>(
+                IntStream.range(0, ARR_LANGUAGES.length)
+                .boxed()
+                .collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i)));
+        public String[] strings = new String[ARR_LANGUAGES.length];
+
+        public TextStrings() {};
+
+        public TextStrings(String init) {
+            for (int i = 0; i < NUM_LANGUAGES; i++)
+                this.strings[i] = init;
+        };
+
+        public TextStrings(List<String> strings, int key) {
+            // Some hashes don't have strings for some languages :(
+            String nullReplacement = "[N/A] %d".formatted((long) key & 0xFFFFFFFFL);
+            for (int i = 0; i < NUM_LANGUAGES; i++) {  // Find first non-null if there is any
+                String s = strings.get(i);
+                if (s != null) {
+                    nullReplacement = "[%s] - %s".formatted(ARR_LANGUAGES[i], s);
+                    break;
+                }
+            }
+            for (int i = 0; i < NUM_LANGUAGES; i++) {
+                String s = strings.get(i);
+                if (s != null)
+                    this.strings[i] = s;
+                else
+                    this.strings[i] = nullReplacement;
+            }
+        }
+
+        public String get(String languageCode) {
+            return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)];
+        }
+
+        public boolean set(String languageCode, String string) {
+            int index = MAP_LANGUAGES.getOrDefault(languageCode, -1);
+            if (index < 0) return false;
+            strings[index] = string;
+            return true;
+        }
+    }
+
+    private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\"");
+
+    private static Int2ObjectMap<String> loadTextMapFile(String language, IntSet nameHashes) {
+        Int2ObjectMap<String> output = new Int2ObjectOpenHashMap<>();
+        try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) {
+            Matcher matcher = textMapKeyValueRegex.matcher("");
+            return new Int2ObjectOpenHashMap<>(
+                file.lines()
+                    .sequential()
+                    .map(matcher::reset)  // Side effects, but it's faster than making a new one
+                    .filter(Matcher::find)
+                    .filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1))))  // TODO: Cache this parse somehow
+                    .collect(Collectors.toMap(
+                        m -> (int) Long.parseLong(m.group(1)),
+                        m -> m.group(2).replace("\\\"", "\""))));
+        } catch (Exception e) {
+            Grasscutter.getLogger().error("Error loading textmap: " + language);
+            Grasscutter.getLogger().error(e.toString());
+        }
+        return output;
+    }
+
+    private static Int2ObjectMap<TextStrings> loadTextMapFiles(IntSet nameHashes) {
+        Map<Integer, Int2ObjectMap<String>> mapLanguageMaps =  // Separate step to process the textmaps in parallel
+            TextStrings.LIST_LANGUAGES.parallelStream().collect(
+            Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMapFile(s, nameHashes)));
+        List<Int2ObjectMap<String>> languageMaps = 
+            IntStream.range(0, TextStrings.NUM_LANGUAGES)
+            .mapToObj(i -> mapLanguageMaps.get(i))
+            .collect(Collectors.toList());
+
+        Map<TextStrings, TextStrings> canonicalTextStrings = new HashMap<>();
+        return new Int2ObjectOpenHashMap<TextStrings>(
+            nameHashes
+            .intStream()
+            .boxed()
+            .collect(Collectors.toMap(key -> key, key -> {
+                TextStrings t = new TextStrings(
+                    IntStream.range(0, TextStrings.NUM_LANGUAGES)
+                    .mapToObj(i -> languageMaps.get(i).get((int) key))
+                    .collect(Collectors.toList()), (int) key);
+                return canonicalTextStrings.computeIfAbsent(t, x -> t);
+                }))
+            );
+    }
+
+    private static Int2ObjectMap<TextStrings> loadTextMapsCache() throws Exception {
+        try (ObjectInputStream file = new ObjectInputStream(new BufferedInputStream(Files.newInputStream(TEXTMAP_CACHE_PATH), 0x100000))) {
+            final int fileVersion = file.readInt();
+            if (fileVersion != TEXTMAP_CACHE_VERSION)
+                throw new Exception("Invalid cache version");
+            return (Int2ObjectMap<TextStrings>) file.readObject();
+        }
+    }
+
+    private static void saveTextMapsCache(Int2ObjectMap<TextStrings> input) throws IOException {
+        try {
+            Files.createDirectory(Path.of("cache"));
+        } catch (FileAlreadyExistsException ignored) {};
+        try (ObjectOutputStream file = new ObjectOutputStream(new BufferedOutputStream(Files.newOutputStream(TEXTMAP_CACHE_PATH, StandardOpenOption.CREATE), 0x100000))) {
+            file.writeInt(TEXTMAP_CACHE_VERSION);
+            file.writeObject(input);
+        }
+    }
+
+    private static Int2ObjectMap<TextStrings> textMapStrings;
+    private static final Path TEXTMAP_CACHE_PATH = Path.of(Utils.toFilePath("cache/TextMapCache.bin"));
+
+    public static Int2ObjectMap<TextStrings> getTextMapStrings() {
+        if (textMapStrings == null)
+            loadTextMaps();
+        return textMapStrings;
+    }
+
+    public static TextStrings getTextMapKey(long hash) {
+        return textMapStrings.get((int) hash);
+    }
+
+    public static void loadTextMaps() {
+        // Check system timestamps on cache and resources
+        try {
+            long cacheModified = Files.getLastModifiedTime(TEXTMAP_CACHE_PATH).toMillis();
+
+            long textmapsModified = Files.list(Path.of(RESOURCE("TextMap")))
+                .filter(path -> path.toString().endsWith(".json"))
+                .map(path -> {
+                    try {
+                        return Files.getLastModifiedTime(path).toMillis();
+                    } catch (Exception ignored) {
+                        Grasscutter.getLogger().debug("Exception while checking modified time: ", path);
+                        return Long.MAX_VALUE;  // Don't use cache, something has gone wrong
+                    }
+                })
+                .max(Long::compare)
+                .get();
+
+                Grasscutter.getLogger().debug("Cache modified %d, textmap modified %d".formatted(cacheModified, textmapsModified));
+            if (textmapsModified < cacheModified) {
+                // Try loading from cache
+                Grasscutter.getLogger().info("Loading cached TextMaps");
+                textMapStrings = loadTextMapsCache();
+                return;
+            }
+        } catch (Exception e) {
+            Grasscutter.getLogger().debug("Exception while checking cache: ", e);
+        };
+
+        // Regenerate cache
+        Grasscutter.getLogger().info("Generating TextMaps cache");
+        ResourceLoader.loadAll();
+        IntSet usedHashes = new IntOpenHashSet();
+        GameData.getAvatarDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
+        GameData.getItemDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
+        GameData.getMonsterDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
+        GameData.getMainQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getTitleTextMapHash()));
+        GameData.getQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getDescTextMapHash()));
+        // Incidental strings
+        usedHashes.add((int) 4233146695L);  // Character
+        usedHashes.add((int) 4231343903L);  // Weapon
+        usedHashes.add((int)  332935371L);  // Standard Wish
+        usedHashes.add((int) 2272170627L);  // Character Event Wish
+        usedHashes.add((int) 3352513147L);  // Character Event Wish-2
+        usedHashes.add((int) 2864268523L);  // Weapon Event Wish
+
+        textMapStrings = loadTextMapFiles(usedHashes);
+        try {
+            saveTextMapsCache(textMapStrings);
+        } catch (IOException e) {
+            Grasscutter.getLogger().error("Failed to save TextMap cache: ", e);
+        };
+    }
 }