Change card db handling (#286)
* reorganize card db, add doit file * make card_db_src a true source by removing untranslated entries, removing the untranslated field * turn update language script into a better compiler * move tools into submodule * remove language db comparison test; add travis compile and check-in of card db Co-authored-by: Nick Vance <nickv2002@users.noreply.github.com>
This commit is contained in:
Peter authored and GitHub committed
1 parent
6fd593f1b6
commit
69c0a66868
57 files changed
+25019
-4677
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,115 @@
|
||||
import os
|
||||
import codecs
|
||||
import json
|
||||
import sys
|
||||
import io
|
||||
import csv
|
||||
from shutil import copyfile
|
||||
|
||||
LANGUAGE_DEFAULT = "en_us" # default language, which takes priority
|
||||
LANGUAGE_XX = "xx" # language for starting a translation
|
||||
|
||||
card_db_dir = os.path.join("..", "domdiv", "card_db") # directory of card data
|
||||
output_dir = os.path.join(".", "card_db") # directory for output data
|
||||
crossReference = "./CrossReference"
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
LANGUAGE_NEW = sys.argv[1].strip().lower()
|
||||
else:
|
||||
print("Usage: ", sys.argv[0], " xx")
|
||||
print("where xx is the two letter language abreviation.")
|
||||
sys.exit()
|
||||
|
||||
|
||||
def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
|
||||
csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
|
||||
for row in csv_reader:
|
||||
yield [cell.decode("iso-8859-15") for cell in row]
|
||||
|
||||
|
||||
def get_json_data(json_file_path):
|
||||
# Read in the json from the specified file
|
||||
with codecs.open(json_file_path, "r", "utf-8") as json_file:
|
||||
data = json.load(json_file)
|
||||
assert data, "Could not load json at: '%r' " % json_file_path
|
||||
return data
|
||||
|
||||
|
||||
def json_dict_entry(entry, separator=""):
|
||||
# Return a nicely formated json dict entry.
|
||||
# It does not include the enclosing {} and removes trailing white space
|
||||
json_data = json.dumps(entry, indent=4, ensure_ascii=False, sort_keys=True)
|
||||
json_data = json_data.strip(
|
||||
"{}"
|
||||
).rstrip() # Remove outer{} and then trailing whitespace
|
||||
return separator + json_data
|
||||
|
||||
|
||||
def find_index(items, find):
|
||||
if find in items:
|
||||
return items.index(find)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
reader = unicode_csv_reader(open(crossReference + ".csv"))
|
||||
cards = list(reader)
|
||||
headers = cards[0]
|
||||
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
lang_dir = os.path.join(output_dir, LANGUAGE_NEW)
|
||||
if not os.path.exists(lang_dir):
|
||||
os.makedirs(lang_dir)
|
||||
|
||||
for f in ["bonuses_", "cards_", "sets_", "types_"]:
|
||||
copyfile(
|
||||
os.path.join(card_db_dir, LANGUAGE_XX, f + LANGUAGE_XX + ".json"),
|
||||
os.path.join(output_dir, LANGUAGE_NEW, f + LANGUAGE_NEW + ".json"),
|
||||
)
|
||||
|
||||
|
||||
lang_index = find_index(headers, LANGUAGE_NEW)
|
||||
card_index = find_index(headers, "card_tag")
|
||||
|
||||
if lang_index is None or card_index is None:
|
||||
print("Cound not find new language '", LANGUAGE_NEW, "' in the CrossReference file")
|
||||
print("Files copied, but no name changes were made.")
|
||||
sys.exit()
|
||||
|
||||
# Get the new names for those in the CrossReference
|
||||
cardlist = {}
|
||||
for row in cards:
|
||||
if row[lang_index]:
|
||||
cardlist[row[card_index]] = row[lang_index]
|
||||
|
||||
# Update the names based upon names in the cross reference file
|
||||
fname = os.path.join(card_db_dir, LANGUAGE_XX, "cards_" + LANGUAGE_XX + ".json")
|
||||
if os.path.isfile(fname):
|
||||
lang_data = get_json_data(fname)
|
||||
else:
|
||||
print("ERROR: Failed to open :", fname)
|
||||
sys.exit()
|
||||
|
||||
if cardlist:
|
||||
with io.open(
|
||||
os.path.join(output_dir, LANGUAGE_NEW, "cards_" + LANGUAGE_NEW + ".json"),
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
) as lang_out:
|
||||
|
||||
lang_out.write("{") # Start of set
|
||||
sep = ""
|
||||
|
||||
for card in lang_data:
|
||||
# print card, lang_data[card]
|
||||
data = lang_data[card]
|
||||
if card in cardlist:
|
||||
# Have a new name, so update name
|
||||
data["name"] = cardlist[card]
|
||||
fields = ["description", "extra"] # but no u"name"
|
||||
data["untranslated"] = fields
|
||||
lang_out.write(json_dict_entry({card: data}, sep))
|
||||
sep = ","
|
||||
lang_out.write("\n}\n") # End of Set
|
||||
@@ -0,0 +1,13 @@
|
||||
import sys
|
||||
|
||||
from domdiv.tools.common import get_languages, load_language_cards
|
||||
|
||||
|
||||
def check_languages(card_db_dir):
|
||||
languages = get_languages(card_db_dir)
|
||||
for lang in languages:
|
||||
_ = load_language_cards(lang, card_db_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_languages(sys.argv[1])
|
||||
@@ -0,0 +1,95 @@
|
||||
import codecs
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
|
||||
LANGUAGE_DEFAULT = "en_us" # default language, which takes priority
|
||||
LANGUAGE_XX = "xx" # language for starting a translation
|
||||
|
||||
|
||||
# Multikey sort
|
||||
# see: http://stackoverflow.com/questions/1143671/python-sorting-list-of-dictionaries-by-multiple-keys
|
||||
def multikeysort(items, columns):
|
||||
from operator import itemgetter
|
||||
|
||||
for c in columns[::-1]:
|
||||
items = sorted(items, key=itemgetter(c))
|
||||
return items
|
||||
|
||||
|
||||
def get_languages(card_db_dir):
|
||||
languages = get_lang_dirs(card_db_dir)
|
||||
languages.remove(LANGUAGE_DEFAULT)
|
||||
languages.insert(0, LANGUAGE_DEFAULT)
|
||||
if LANGUAGE_XX not in languages:
|
||||
languages.append(LANGUAGE_XX)
|
||||
return languages
|
||||
|
||||
|
||||
def get_lang_dirs(path):
|
||||
# Find all valid languages.
|
||||
lang_dirs = []
|
||||
for name in os.listdir(path):
|
||||
dir_path = os.path.join(path, name)
|
||||
if os.path.isdir(dir_path):
|
||||
cards_file = os.path.join(dir_path, "cards_" + name + ".json")
|
||||
sets_file = os.path.join(dir_path, "sets_" + name + ".json")
|
||||
if os.path.isfile(cards_file) and os.path.isfile(sets_file):
|
||||
lang_dirs.append(name)
|
||||
return lang_dirs
|
||||
|
||||
|
||||
def get_json_data(json_file_path):
|
||||
print(("reading {}".format(json_file_path)))
|
||||
# Read in the json from the specified file
|
||||
with codecs.open(json_file_path, "r", "utf-8") as json_file:
|
||||
data = json.load(json_file)
|
||||
assert data, "Could not load json at: '%r' " % json_file_path
|
||||
return data
|
||||
|
||||
|
||||
def load_language_cards(lang, card_db_dir):
|
||||
# contruct the cards json file name
|
||||
lang_file = f"cards_{lang}.json"
|
||||
fname = os.path.join(card_db_dir, lang, lang_file)
|
||||
if os.path.isfile(fname):
|
||||
lang_data = get_json_data(fname)
|
||||
else:
|
||||
lang_data = {}
|
||||
return lang_data
|
||||
|
||||
|
||||
def write_data(data, fname):
|
||||
# Process the file
|
||||
print(f"writing {fname}")
|
||||
with io.open(os.path.join(fname), "w", encoding="utf-8") as lang_out:
|
||||
json.dump(data, lang_out, indent=4, ensure_ascii=False)
|
||||
lang_out.write("\n")
|
||||
|
||||
|
||||
def write_language_cards(cards, lang, card_db_dir):
|
||||
lang_file = f"cards_{lang}.json"
|
||||
fname = os.path.join(card_db_dir, lang, lang_file)
|
||||
write_data(cards, fname)
|
||||
|
||||
|
||||
def load_card_data(card_db_dir):
|
||||
###########################################################################
|
||||
# Get the cards_db information
|
||||
# Store in a list in the order found in cards[]. Ordered as follows:
|
||||
# 1. card_tags, 2. group_tags, 3. super groups
|
||||
###########################################################################
|
||||
|
||||
# Get the card data
|
||||
card_data = get_json_data(os.path.join(card_db_dir, "cards_db.json"))
|
||||
|
||||
# Sort the cardset_tags
|
||||
for card in card_data:
|
||||
card["cardset_tags"].sort()
|
||||
# But put all the base cards together by moving to front of the list
|
||||
if "base" in card["cardset_tags"]:
|
||||
card["cardset_tags"].remove("base")
|
||||
card["cardset_tags"].insert(0, "base")
|
||||
|
||||
# Sort the cards by cardset_tags, then card_tag
|
||||
return multikeysort(card_data, ["cardset_tags", "card_tag"])
|
||||
@@ -0,0 +1,48 @@
|
||||
# coding=utf-8
|
||||
|
||||
import csv
|
||||
import sys
|
||||
import json
|
||||
|
||||
converted = []
|
||||
|
||||
typemap = {
|
||||
"Aktion": "Action",
|
||||
"Geld": "Treasure",
|
||||
"Fluch": "Curse",
|
||||
"Punkte": "Victory",
|
||||
"Reaktion": "Reaction",
|
||||
"Angriff": "Attack",
|
||||
"Dauer": "Duration",
|
||||
"Plündern": "Looter",
|
||||
"Ritter": "Knight",
|
||||
"Ruine": "Ruins",
|
||||
"Unterschlupf": "Shelter",
|
||||
"Reisender": "Traveller",
|
||||
"Reserve": "Reserve",
|
||||
}
|
||||
|
||||
with open(sys.argv[1], "U") as csvfile:
|
||||
reader = csv.DictReader(csvfile)
|
||||
for row in reader:
|
||||
if not row["Typ"]:
|
||||
continue
|
||||
# row = {k: v.decode('ISO-8859-2').encode('utf-8') if type(v) == str else v for k, v in row.iteritems()}
|
||||
# print chardet.detect(row['Kartentext'])
|
||||
if "Ritter" in row["Typ"] and row["Kartenname"] != "Ritter":
|
||||
continue
|
||||
converted_row = {
|
||||
"name": row["Kartenname"],
|
||||
"cost": row["Kosten"],
|
||||
"cardset": row["Edition"],
|
||||
"description": row["Kartentext"],
|
||||
"extra": row["Lange Erklärung"],
|
||||
"types": [
|
||||
typemap[t.strip()]
|
||||
for t in row["Typ"].split("/")
|
||||
if not t.strip() == "Ritter"
|
||||
],
|
||||
"potcost": row.get("potcost", 0),
|
||||
}
|
||||
converted.append(converted_row)
|
||||
json.dump(converted, open(sys.argv[2], "wb"), indent=True)
|
||||
@@ -0,0 +1,297 @@
|
||||
###########################################################################
|
||||
# This file provides maintenance on the various language files
|
||||
# 1. Create new "xx/cards_xx.json" files that have entries ordered as:
|
||||
# a. the card_tag entries in "cards_db.json"
|
||||
# b. the group_tag entries as found in "cards_db.json"
|
||||
# c. the super group entries (grouping across all expansions"
|
||||
# d. any unused entries existing in the file (assumed to be work in progress)
|
||||
#
|
||||
# 2. Create new "sets_db.json" and "xx/cards_xx.json" with entries sorted alphabetically
|
||||
#
|
||||
# All output is in the designated output directory. Original files are not overwritten.
|
||||
###########################################################################
|
||||
|
||||
import os
|
||||
import os.path
|
||||
from shutil import copyfile
|
||||
import argparse
|
||||
import collections
|
||||
|
||||
from domdiv.tools.common import (
|
||||
get_json_data,
|
||||
load_language_cards,
|
||||
LANGUAGE_XX,
|
||||
LANGUAGE_DEFAULT,
|
||||
get_languages,
|
||||
multikeysort,
|
||||
load_card_data,
|
||||
write_data,
|
||||
write_language_cards,
|
||||
)
|
||||
|
||||
|
||||
def main(card_db_dir, output_dir):
|
||||
###########################################################################
|
||||
# Get all the languages, and place the default language first in the list
|
||||
###########################################################################
|
||||
languages = get_languages(card_db_dir)
|
||||
print("Languages:")
|
||||
print(languages)
|
||||
print()
|
||||
|
||||
###########################################################################
|
||||
# Make sure the directories exist to hold the output
|
||||
###########################################################################
|
||||
|
||||
# main output directory
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
# each language directory
|
||||
for lang in languages:
|
||||
# Make sure the directory is there to hold the file
|
||||
lang_dir = os.path.join(output_dir, lang)
|
||||
if not os.path.exists(lang_dir):
|
||||
os.makedirs(lang_dir)
|
||||
|
||||
###########################################################################
|
||||
# Get the types_db information
|
||||
# Store in a list in the order found in types[]. Ordered by card_type
|
||||
# 1. card_tags, 2. group_tags, 3. super groups
|
||||
###########################################################################
|
||||
type_parts = set()
|
||||
|
||||
# Get the card data
|
||||
type_data = get_json_data(os.path.join(card_db_dir, "types_db.json"))
|
||||
|
||||
# Sort the cards by cardset_tags, then card_tag
|
||||
sorted_type_data = multikeysort(type_data, ["card_type"])
|
||||
|
||||
write_data(sorted_type_data, os.path.join(output_dir, "types_db.json"))
|
||||
|
||||
type_parts = list(set().union(*[set(t["card_type"]) for t in sorted_type_data]))
|
||||
type_parts.sort()
|
||||
print("Unique Types:")
|
||||
print(type_parts)
|
||||
print()
|
||||
|
||||
###########################################################################
|
||||
# Get the labels_db information
|
||||
# Store in a list in the order found.
|
||||
###########################################################################
|
||||
all_labels = []
|
||||
|
||||
# Get the card data
|
||||
label_data = get_json_data(os.path.join(card_db_dir, "labels_db.json"))
|
||||
|
||||
all_labels = list(set().union(*[set(label["names"]) for label in label_data]))
|
||||
write_data(label_data, os.path.join(output_dir, "labels_db.json"))
|
||||
|
||||
all_labels.sort()
|
||||
print("Labels: ")
|
||||
print(all_labels)
|
||||
print()
|
||||
###########################################################################
|
||||
# Fix up all the xx/types_xx.json files
|
||||
# Place entries in alphabetical order
|
||||
# If entries don't exist:
|
||||
# If the default language, set from information in the "types_db.json" file,
|
||||
# If not the default language, set based on information from the default language.
|
||||
# Lastly, keep any extra entries that are not currently used, just in case needed
|
||||
# in the future or is a work in progress.
|
||||
###########################################################################
|
||||
for lang in languages:
|
||||
lang_file = "types_" + lang + ".json"
|
||||
fname = os.path.join(card_db_dir, lang, lang_file)
|
||||
if os.path.isfile(fname):
|
||||
lang_type_data = get_json_data(fname)
|
||||
else:
|
||||
lang_type_data = {}
|
||||
|
||||
for t in sorted(type_parts):
|
||||
if t not in lang_type_data:
|
||||
if lang == LANGUAGE_DEFAULT:
|
||||
lang_type_data[t] = t
|
||||
lang_type_default = lang_type_data
|
||||
else:
|
||||
lang_type_data[t] = lang_type_default[t]
|
||||
write_data(lang_type_data, os.path.join(output_dir, lang, lang_file))
|
||||
|
||||
if lang == LANGUAGE_DEFAULT:
|
||||
lang_type_default = lang_type_data # Keep for later languages
|
||||
|
||||
sorted_card_data = load_card_data(card_db_dir)
|
||||
groups = set(card["group_tag"] for card in sorted_card_data if "group_tag" in card)
|
||||
super_groups = set(["events", "landmarks"])
|
||||
|
||||
write_data(sorted_card_data, os.path.join(output_dir, "cards_db.json"))
|
||||
|
||||
# maintain the sorted order, but expand with groups and super_groups
|
||||
cards = [c["card_tag"] for c in sorted_card_data]
|
||||
cards.extend(sorted(groups))
|
||||
cards.extend(sorted(super_groups))
|
||||
|
||||
print("Cards:")
|
||||
print(cards)
|
||||
print()
|
||||
|
||||
###########################################################################
|
||||
# Fix up all the cards_xx.json files
|
||||
# Place entries in the same order as given in "cards_db.json".
|
||||
# If entries don't exist:
|
||||
# If the default language, set base on information in the "cards_db.json" file,
|
||||
# If not the default language, set based on information from the default language.
|
||||
# Lastly, keep any extra entries that are not currently used, just in case needed
|
||||
# in the future or is a work in progress.
|
||||
###########################################################################
|
||||
for lang in languages:
|
||||
|
||||
# contruct the cards json file name
|
||||
lang_data = load_language_cards(lang, card_db_dir)
|
||||
|
||||
sorted_lang_data = collections.OrderedDict()
|
||||
for card_tag in cards:
|
||||
lang_card = lang_data.get(card_tag)
|
||||
|
||||
# print(f'looking at {card_tag}: {lang_card}')
|
||||
if not lang_card or lang == LANGUAGE_XX:
|
||||
# Card is missing, need to add it
|
||||
lang_card = {}
|
||||
if lang == LANGUAGE_DEFAULT:
|
||||
# Default language gets bare minimum. Really need to add by hand.
|
||||
lang_card["extra"] = ""
|
||||
lang_card["name"] = card_tag
|
||||
lang_card["description"] = ""
|
||||
lang_default = lang_data
|
||||
else:
|
||||
# All other languages should get the default languages' text
|
||||
lang_card = lang_default[card_tag].copy()
|
||||
elif lang != LANGUAGE_DEFAULT:
|
||||
# Card exists, figure out what needs updating
|
||||
lang_card.update(
|
||||
{
|
||||
field: value
|
||||
for field, value in lang_default[card_tag].items()
|
||||
if field not in lang_card
|
||||
}
|
||||
)
|
||||
sorted_lang_data[card_tag] = lang_card
|
||||
unused = set(lang_data) - set(sorted_lang_data)
|
||||
print(
|
||||
f"unused in {lang}: {len(unused)}, used: {len(set(lang_data) & set(sorted_lang_data))}"
|
||||
)
|
||||
print(unused)
|
||||
# Now keep any unused values just in case needed in the future
|
||||
for card_tag in sorted(unused):
|
||||
lang_card = lang_data.get(card_tag)
|
||||
lang_card["notes"] = ["This card is currently not used."]
|
||||
sorted_lang_data[card_tag] = lang_card
|
||||
|
||||
write_language_cards(sorted_lang_data, lang, output_dir)
|
||||
|
||||
if lang == LANGUAGE_DEFAULT:
|
||||
lang_default = lang_data # Keep for later languages
|
||||
|
||||
###########################################################################
|
||||
# Fix up the sets_db.json file
|
||||
# Place entries in alphabetical order
|
||||
###########################################################################
|
||||
lang_file = "sets_db.json"
|
||||
set_data = get_json_data(os.path.join(card_db_dir, lang_file))
|
||||
|
||||
write_data(set_data, os.path.join(output_dir, lang_file))
|
||||
|
||||
print("Sets:")
|
||||
print(set(set_data))
|
||||
print()
|
||||
|
||||
###########################################################################
|
||||
# Fix up all the xx/sets_xx.json files
|
||||
# Place entries in alphabetical order
|
||||
# If entries don't exist:
|
||||
# If the default language, set from information in the "sets_db.json" file,
|
||||
# If not the default language, set based on information from the default language.
|
||||
###########################################################################
|
||||
for lang in languages:
|
||||
lang_file = "sets_" + lang + ".json"
|
||||
fname = os.path.join(card_db_dir, lang, lang_file)
|
||||
if os.path.isfile(fname):
|
||||
lang_set_data = get_json_data(fname)
|
||||
else:
|
||||
lang_set_data = {}
|
||||
|
||||
for s in sorted(set_data):
|
||||
if s not in lang_set_data:
|
||||
lang_set_data[s] = {}
|
||||
if lang == LANGUAGE_DEFAULT:
|
||||
lang_set_data[s]["set_name"] = s.title()
|
||||
lang_set_data[s]["text_icon"] = set_data[s]["text_icon"]
|
||||
if "short_name" in set_data[s]:
|
||||
lang_set_data[s]["short_name"] = set_data[s]["short_name"]
|
||||
if "set_text" in set_data[s]:
|
||||
lang_set_data[s]["set_text"] = set_data[s]["set_text"]
|
||||
else:
|
||||
lang_set_data[s]["set_name"] = lang_default[s]["set_name"]
|
||||
lang_set_data[s]["text_icon"] = lang_default[s]["text_icon"]
|
||||
if "short_name" in lang_default[s]:
|
||||
lang_set_data[s]["short_name"] = lang_default[s]["short_name"]
|
||||
if "set_text" in lang_default[s]:
|
||||
lang_set_data[s]["set_text"] = lang_default[s]["set_text"]
|
||||
else:
|
||||
if lang != LANGUAGE_DEFAULT:
|
||||
for x in lang_default[s]:
|
||||
if x not in lang_set_data[s] and x != "used":
|
||||
lang_set_data[s][x] = lang_default[s][x]
|
||||
|
||||
if lang == LANGUAGE_DEFAULT:
|
||||
lang_default = lang_set_data # Keep for later languages
|
||||
|
||||
write_data(lang_set_data, os.path.join(output_dir, lang, lang_file))
|
||||
|
||||
###########################################################################
|
||||
# bonuses_xx files
|
||||
###########################################################################
|
||||
for lang in languages:
|
||||
# Special case for xx. Reseed from default language
|
||||
fromLanguage = lang
|
||||
if lang == LANGUAGE_XX:
|
||||
fromLanguage = LANGUAGE_DEFAULT
|
||||
|
||||
copyfile(
|
||||
os.path.join(
|
||||
card_db_dir, fromLanguage, "bonuses_" + fromLanguage + ".json"
|
||||
),
|
||||
os.path.join(output_dir, lang, "bonuses_" + lang + ".json"),
|
||||
)
|
||||
|
||||
###########################################################################
|
||||
# translation.txt
|
||||
###########################################################################
|
||||
copyfile(
|
||||
os.path.join(card_db_dir, "translation.md"),
|
||||
os.path.join(output_dir, "translation.md"),
|
||||
)
|
||||
|
||||
# Since xx is the starting point for new translations,
|
||||
# make sure xx has the latest copy of translation.txt
|
||||
copyfile(
|
||||
os.path.join(card_db_dir, LANGUAGE_XX, "translation.txt"),
|
||||
os.path.join(output_dir, LANGUAGE_XX, "translation.txt"),
|
||||
)
|
||||
|
||||
|
||||
def run():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"card_db_dir",
|
||||
help="directory of card data (usually card_db_src at the top level of the distribution)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"output_dir", help="directory for output data (usually src/domdiv/card_db)"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args.card_db_dir, args.output_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Reference in new issue
Block a user