mirror of
https://github.com/Melledy/Grasscutter.git
synced 2024-11-24 14:39:24 +00:00
Cache used strings from TextMaps
This commit is contained in:
AnimeGitB
Luke H-W
parent
4790158ac6
commit
0994417261
1
.gitignore
vendored
1
.gitignore
vendored
@ -52,6 +52,7 @@ tmp/
|
||||
.vscode
|
||||
|
||||
# Grasscutter
|
||||
/cache
|
||||
/resources
|
||||
/logs
|
||||
/plugins
|
||||
|
||||
@ -53,7 +53,9 @@ public class ResourceLoader {
|
||||
return classList;
|
||||
}
|
||||
|
||||
private static boolean loadedAll = false;
|
||||
public static void loadAll() {
|
||||
if (loadedAll) return;
|
||||
Grasscutter.getLogger().info(translate("messages.status.resources.loading"));
|
||||
|
||||
// Load ability lists
|
||||
@ -75,6 +77,7 @@ public class ResourceLoader {
|
||||
loadNpcBornData();
|
||||
|
||||
Grasscutter.getLogger().info(translate("messages.status.resources.finish"));
|
||||
loadedAll = true;
|
||||
}
|
||||
|
||||
public static void loadResources() {
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
package emu.grasscutter.tools;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.PrintWriter;
|
||||
@ -12,10 +10,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
|
||||
@ -33,119 +28,25 @@ import emu.grasscutter.data.excels.QuestData;
|
||||
import emu.grasscutter.data.excels.SceneData;
|
||||
import emu.grasscutter.utils.Language;
|
||||
import emu.grasscutter.utils.Utils;
|
||||
import emu.grasscutter.utils.Language.TextStrings;
|
||||
import it.unimi.dsi.fastutil.ints.Int2IntMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.ints.IntArrayList;
|
||||
import it.unimi.dsi.fastutil.ints.IntList;
|
||||
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
|
||||
import it.unimi.dsi.fastutil.ints.IntSet;
|
||||
import it.unimi.dsi.fastutil.objects.Object2IntMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import static emu.grasscutter.config.Configuration.*;
|
||||
import static emu.grasscutter.utils.Language.translate;
|
||||
|
||||
public final class Tools {
|
||||
@EqualsAndHashCode public static class TextStrings {
|
||||
public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"};
|
||||
public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"};
|
||||
public static final int NUM_LANGUAGES = ARR_LANGUAGES.length;
|
||||
public static final List<String> LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES);
|
||||
public static final Object2IntMap<String> MAP_LANGUAGES = // Map "EN": 0, "CHS": 1, ..., "VI": 12
|
||||
new Object2IntOpenHashMap<>(
|
||||
IntStream.range(0, ARR_LANGUAGES.length)
|
||||
.boxed()
|
||||
.collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i)));
|
||||
public String[] strings = new String[ARR_LANGUAGES.length];
|
||||
|
||||
public TextStrings() {};
|
||||
|
||||
public TextStrings(String init) {
|
||||
for (int i = 0; i < NUM_LANGUAGES; i++)
|
||||
this.strings[i] = init;
|
||||
};
|
||||
|
||||
public TextStrings(Collection<String> strings) {
|
||||
this.strings = strings.toArray(new String[0]);
|
||||
}
|
||||
|
||||
public String get(String languageCode) {
|
||||
return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)];
|
||||
}
|
||||
|
||||
public boolean set(String languageCode, String string) {
|
||||
int index = MAP_LANGUAGES.getOrDefault(languageCode, -1);
|
||||
if (index < 0) return false;
|
||||
strings[index] = string;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\"");
|
||||
|
||||
private static Int2ObjectMap<String> loadTextMap(String language, IntSet nameHashes) {
|
||||
Int2ObjectMap<String> output = new Int2ObjectOpenHashMap<>();
|
||||
try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) {
|
||||
Matcher matcher = textMapKeyValueRegex.matcher("");
|
||||
return new Int2ObjectOpenHashMap<>(
|
||||
file.lines()
|
||||
.sequential()
|
||||
.map(matcher::reset) // Side effects, but it's faster than making a new one
|
||||
.filter(Matcher::find)
|
||||
.filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1)))) // TODO: Cache this parse somehow
|
||||
.collect(Collectors.toMap(
|
||||
m -> (int) Long.parseLong(m.group(1)),
|
||||
m -> m.group(2))));
|
||||
} catch (Exception e) {
|
||||
Grasscutter.getLogger().error("Error loading textmap: " + language);
|
||||
Grasscutter.getLogger().error(e.toString());
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
public static Int2ObjectMap<TextStrings> loadTextMaps(IntSet nameHashes) {
|
||||
Map<Integer, Int2ObjectMap<String>> mapLanguageMaps = // Separate step to process the textmaps in parallel
|
||||
TextStrings.LIST_LANGUAGES.parallelStream().collect(
|
||||
Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMap(s, nameHashes)));
|
||||
List<Int2ObjectMap<String>> languageMaps =
|
||||
IntStream.range(0, TextStrings.NUM_LANGUAGES)
|
||||
.mapToObj(i -> mapLanguageMaps.get(i))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
Map<TextStrings, TextStrings> canonicalTextStrings = new HashMap<>();
|
||||
return new Int2ObjectOpenHashMap<TextStrings>(
|
||||
nameHashes
|
||||
.intStream()
|
||||
.boxed()
|
||||
.collect(Collectors.toMap(key -> key, key -> {
|
||||
TextStrings t = new TextStrings(
|
||||
IntStream.range(0, TextStrings.NUM_LANGUAGES)
|
||||
.mapToObj(i -> languageMaps.get(i).getOrDefault((int) key, "[N/A] - hash key %d".formatted(key)))
|
||||
.collect(Collectors.toList()));
|
||||
return canonicalTextStrings.computeIfAbsent(t, x -> t);
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
public static void createGmHandbooks() throws Exception {
|
||||
ResourceLoader.loadAll();
|
||||
Int2IntMap avatarNames = new Int2IntOpenHashMap(GameData.getAvatarDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
|
||||
Int2IntMap itemNames = new Int2IntOpenHashMap(GameData.getItemDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
|
||||
Int2IntMap monsterNames = new Int2IntOpenHashMap(GameData.getMonsterDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
|
||||
Int2IntMap mainQuestTitles = new Int2IntOpenHashMap(GameData.getMainQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getTitleTextMapHash())));
|
||||
Int2IntMap questDescs = new Int2IntOpenHashMap(GameData.getQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getDescTextMapHash())));
|
||||
// Int2IntMap questDescs = new Int2IntOpenHashMap(GameData.getQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getDescTextMapHash())));
|
||||
|
||||
IntSet usedHashes = new IntOpenHashSet();
|
||||
usedHashes.addAll(avatarNames.values());
|
||||
usedHashes.addAll(itemNames.values());
|
||||
usedHashes.addAll(monsterNames.values());
|
||||
usedHashes.addAll(mainQuestTitles.values());
|
||||
usedHashes.addAll(questDescs.values());
|
||||
|
||||
Int2ObjectMap<TextStrings> textMaps = loadTextMaps(usedHashes);
|
||||
Int2ObjectMap<TextStrings> textMaps = Language.getTextMapStrings();
|
||||
|
||||
Language savedLanguage = Grasscutter.getLanguage();
|
||||
|
||||
@ -233,10 +134,8 @@ public final class Tools {
|
||||
var questDataMap = GameData.getQuestDataMap();
|
||||
questDataMap.keySet().intStream().sorted().forEach(id -> {
|
||||
QuestData data = questDataMap.get(id);
|
||||
int titleKey = (int) mainQuestTitles.get(data.getMainId());
|
||||
int descKey = (int) data.getDescTextMapHash();
|
||||
TextStrings title = textMaps.get(titleKey);
|
||||
TextStrings desc = textMaps.get(descKey);
|
||||
TextStrings title = textMaps.get((int) mainQuestTitles.get(data.getMainId()));
|
||||
TextStrings desc = textMaps.get((int) data.getDescTextMapHash());
|
||||
for (int i = 0; i < TextStrings.NUM_LANGUAGES; i++) {
|
||||
handbookBuilders[i]
|
||||
.append(id)
|
||||
|
||||
@ -3,14 +3,43 @@ package emu.grasscutter.utils;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import emu.grasscutter.Grasscutter;
|
||||
import emu.grasscutter.data.GameData;
|
||||
import emu.grasscutter.data.ResourceLoader;
|
||||
import emu.grasscutter.game.player.Player;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
|
||||
import it.unimi.dsi.fastutil.ints.IntSet;
|
||||
import it.unimi.dsi.fastutil.objects.Object2IntMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static emu.grasscutter.config.Configuration.*;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.io.Serializable;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public final class Language {
|
||||
@ -207,4 +236,187 @@ public final class Language {
|
||||
return languageFile;
|
||||
}
|
||||
}
|
||||
|
||||
private static final int TEXTMAP_CACHE_VERSION = 0x9CCACE02;
|
||||
@EqualsAndHashCode public static class TextStrings implements Serializable {
|
||||
public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"};
|
||||
public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"};
|
||||
public static final int NUM_LANGUAGES = ARR_LANGUAGES.length;
|
||||
public static final List<String> LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES);
|
||||
public static final Object2IntMap<String> MAP_LANGUAGES = // Map "EN": 0, "CHS": 1, ..., "VI": 12
|
||||
new Object2IntOpenHashMap<>(
|
||||
IntStream.range(0, ARR_LANGUAGES.length)
|
||||
.boxed()
|
||||
.collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i)));
|
||||
public String[] strings = new String[ARR_LANGUAGES.length];
|
||||
|
||||
public TextStrings() {};
|
||||
|
||||
public TextStrings(String init) {
|
||||
for (int i = 0; i < NUM_LANGUAGES; i++)
|
||||
this.strings[i] = init;
|
||||
};
|
||||
|
||||
public TextStrings(List<String> strings, int key) {
|
||||
// Some hashes don't have strings for some languages :(
|
||||
String nullReplacement = "[N/A] %d".formatted((long) key & 0xFFFFFFFFL);
|
||||
for (int i = 0; i < NUM_LANGUAGES; i++) { // Find first non-null if there is any
|
||||
String s = strings.get(i);
|
||||
if (s != null) {
|
||||
nullReplacement = "[%s] - %s".formatted(ARR_LANGUAGES[i], s);
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < NUM_LANGUAGES; i++) {
|
||||
String s = strings.get(i);
|
||||
if (s != null)
|
||||
this.strings[i] = s;
|
||||
else
|
||||
this.strings[i] = nullReplacement;
|
||||
}
|
||||
}
|
||||
|
||||
public String get(String languageCode) {
|
||||
return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)];
|
||||
}
|
||||
|
||||
public boolean set(String languageCode, String string) {
|
||||
int index = MAP_LANGUAGES.getOrDefault(languageCode, -1);
|
||||
if (index < 0) return false;
|
||||
strings[index] = string;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\"");
|
||||
|
||||
private static Int2ObjectMap<String> loadTextMapFile(String language, IntSet nameHashes) {
|
||||
Int2ObjectMap<String> output = new Int2ObjectOpenHashMap<>();
|
||||
try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) {
|
||||
Matcher matcher = textMapKeyValueRegex.matcher("");
|
||||
return new Int2ObjectOpenHashMap<>(
|
||||
file.lines()
|
||||
.sequential()
|
||||
.map(matcher::reset) // Side effects, but it's faster than making a new one
|
||||
.filter(Matcher::find)
|
||||
.filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1)))) // TODO: Cache this parse somehow
|
||||
.collect(Collectors.toMap(
|
||||
m -> (int) Long.parseLong(m.group(1)),
|
||||
m -> m.group(2).replace("\\\"", "\""))));
|
||||
} catch (Exception e) {
|
||||
Grasscutter.getLogger().error("Error loading textmap: " + language);
|
||||
Grasscutter.getLogger().error(e.toString());
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
private static Int2ObjectMap<TextStrings> loadTextMapFiles(IntSet nameHashes) {
|
||||
Map<Integer, Int2ObjectMap<String>> mapLanguageMaps = // Separate step to process the textmaps in parallel
|
||||
TextStrings.LIST_LANGUAGES.parallelStream().collect(
|
||||
Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMapFile(s, nameHashes)));
|
||||
List<Int2ObjectMap<String>> languageMaps =
|
||||
IntStream.range(0, TextStrings.NUM_LANGUAGES)
|
||||
.mapToObj(i -> mapLanguageMaps.get(i))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
Map<TextStrings, TextStrings> canonicalTextStrings = new HashMap<>();
|
||||
return new Int2ObjectOpenHashMap<TextStrings>(
|
||||
nameHashes
|
||||
.intStream()
|
||||
.boxed()
|
||||
.collect(Collectors.toMap(key -> key, key -> {
|
||||
TextStrings t = new TextStrings(
|
||||
IntStream.range(0, TextStrings.NUM_LANGUAGES)
|
||||
.mapToObj(i -> languageMaps.get(i).get((int) key))
|
||||
.collect(Collectors.toList()), (int) key);
|
||||
return canonicalTextStrings.computeIfAbsent(t, x -> t);
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
private static Int2ObjectMap<TextStrings> loadTextMapsCache() throws Exception {
|
||||
try (ObjectInputStream file = new ObjectInputStream(new BufferedInputStream(Files.newInputStream(TEXTMAP_CACHE_PATH), 0x100000))) {
|
||||
final int fileVersion = file.readInt();
|
||||
if (fileVersion != TEXTMAP_CACHE_VERSION)
|
||||
throw new Exception("Invalid cache version");
|
||||
return (Int2ObjectMap<TextStrings>) file.readObject();
|
||||
}
|
||||
}
|
||||
|
||||
private static void saveTextMapsCache(Int2ObjectMap<TextStrings> input) throws IOException {
|
||||
try {
|
||||
Files.createDirectory(Path.of("cache"));
|
||||
} catch (FileAlreadyExistsException ignored) {};
|
||||
try (ObjectOutputStream file = new ObjectOutputStream(new BufferedOutputStream(Files.newOutputStream(TEXTMAP_CACHE_PATH, StandardOpenOption.CREATE), 0x100000))) {
|
||||
file.writeInt(TEXTMAP_CACHE_VERSION);
|
||||
file.writeObject(input);
|
||||
}
|
||||
}
|
||||
|
||||
private static Int2ObjectMap<TextStrings> textMapStrings;
|
||||
private static final Path TEXTMAP_CACHE_PATH = Path.of(Utils.toFilePath("cache/TextMapCache.bin"));
|
||||
|
||||
public static Int2ObjectMap<TextStrings> getTextMapStrings() {
|
||||
if (textMapStrings == null)
|
||||
loadTextMaps();
|
||||
return textMapStrings;
|
||||
}
|
||||
|
||||
public static TextStrings getTextMapKey(long hash) {
|
||||
return textMapStrings.get((int) hash);
|
||||
}
|
||||
|
||||
public static void loadTextMaps() {
|
||||
// Check system timestamps on cache and resources
|
||||
try {
|
||||
long cacheModified = Files.getLastModifiedTime(TEXTMAP_CACHE_PATH).toMillis();
|
||||
|
||||
long textmapsModified = Files.list(Path.of(RESOURCE("TextMap")))
|
||||
.filter(path -> path.toString().endsWith(".json"))
|
||||
.map(path -> {
|
||||
try {
|
||||
return Files.getLastModifiedTime(path).toMillis();
|
||||
} catch (Exception ignored) {
|
||||
Grasscutter.getLogger().debug("Exception while checking modified time: ", path);
|
||||
return Long.MAX_VALUE; // Don't use cache, something has gone wrong
|
||||
}
|
||||
})
|
||||
.max(Long::compare)
|
||||
.get();
|
||||
|
||||
Grasscutter.getLogger().debug("Cache modified %d, textmap modified %d".formatted(cacheModified, textmapsModified));
|
||||
if (textmapsModified < cacheModified) {
|
||||
// Try loading from cache
|
||||
Grasscutter.getLogger().info("Loading cached TextMaps");
|
||||
textMapStrings = loadTextMapsCache();
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Grasscutter.getLogger().debug("Exception while checking cache: ", e);
|
||||
};
|
||||
|
||||
// Regenerate cache
|
||||
Grasscutter.getLogger().info("Generating TextMaps cache");
|
||||
ResourceLoader.loadAll();
|
||||
IntSet usedHashes = new IntOpenHashSet();
|
||||
GameData.getAvatarDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
|
||||
GameData.getItemDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
|
||||
GameData.getMonsterDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
|
||||
GameData.getMainQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getTitleTextMapHash()));
|
||||
GameData.getQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getDescTextMapHash()));
|
||||
// Incidental strings
|
||||
usedHashes.add((int) 4233146695L); // Character
|
||||
usedHashes.add((int) 4231343903L); // Weapon
|
||||
usedHashes.add((int) 332935371L); // Standard Wish
|
||||
usedHashes.add((int) 2272170627L); // Character Event Wish
|
||||
usedHashes.add((int) 3352513147L); // Character Event Wish-2
|
||||
usedHashes.add((int) 2864268523L); // Weapon Event Wish
|
||||
|
||||
textMapStrings = loadTextMapFiles(usedHashes);
|
||||
try {
|
||||
saveTextMapsCache(textMapStrings);
|
||||
} catch (IOException e) {
|
||||
Grasscutter.getLogger().error("Failed to save TextMap cache: ", e);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user