From 3762d4df050cbc40434ea8179a4c792b15637bec Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 31 Jul 2020 12:07:01 +0200 Subject: [PATCH 1/5] i18n: refactoring txstats --- ci/txstats.py | 83 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/ci/txstats.py b/ci/txstats.py index 3e85a7c27..0f955421e 100755 --- a/ci/txstats.py +++ b/ci/txstats.py @@ -10,22 +10,51 @@ import sys import argparse -def get_tx_credentials(): - """ - Gets the API token out of the user's .transifexrc (this is supposed - to be secure). - """ - import configparser - import os - txconfig_name = os.path.expanduser("~/.transifexrc") - try: - with open(txconfig_name, "r") as f: - parser = configparser.ConfigParser() - parser.read_file(f) +class TXError(Exception): + pass + + +class TransifexGetter(object): + """ + Get language data from Transifex. + + The object does all the work in __init__, after that + the only relevant data is .languages, a dictionary + of language data. + """ + def __init__(self): + token = self.get_tx_credentials() + if token is None: + raise TXError("Could not get Transifex API token") + + import requests + r = requests.get("https://api.transifex.com/organizations/calamares/projects/calamares/resources/calamares/", auth=("api", token)) + if r.status_code != 200: + raise TXError("Could not get Transifex data from API") + + j = r.json() + self.languages = j["stats"] + + + def get_tx_credentials(self): + """ + Gets the API token out of the user's .transifexrc (this is supposed + to be secure). + """ + import configparser + import os + txconfig_name = os.path.expanduser("~/.transifexrc") + try: + with open(txconfig_name, "r") as f: + parser = configparser.ConfigParser() + parser.read_file(f) + + return parser.get("https://www.transifex.com", "password") + except IOError as e: + return None + + - return parser.get("https://www.transifex.com", "password") - except IOError as e: - return None def output_langs(all_langs, label, filterfunc): """ @@ -48,7 +77,8 @@ def output_langs(all_langs, label, filterfunc): prefix = " " print("%s%s" % (prefix, out)) -def get_tx_stats(token, verbose): + +def get_tx_stats(languages, verbose): """ Does an API request to Transifex with the given API @p token, getting the translation statistics for the main body of texts. Then prints @@ -57,12 +87,6 @@ def get_tx_stats(token, verbose): If @p verbose is True, prints out language stats as well. """ - import requests - - r = requests.get("https://api.transifex.com/organizations/calamares/projects/calamares/resources/calamares/", auth=("api", token)) - if r.status_code != 200: - return 1 - suppressed_languages = ( "es_ES", ) # In Transifex, but not used # Some languages go into the "incomplete" list by definition, # regardless of their completion status: this can have various reasons. @@ -75,9 +99,6 @@ def get_tx_stats(token, verbose): ) all_langs = [] - - j = r.json() - languages = j["stats"] print("# Total %d languages" % len(languages)) for lang_name in languages: if lang_name in suppressed_languages: @@ -105,12 +126,12 @@ def main(): parser = argparse.ArgumentParser(description="Update Transifex Statistics") parser.add_argument("--verbose", "-v", help="Show statistics", action="store_true") args = parser.parse_args() - cred = get_tx_credentials() - if cred: - return get_tx_stats(cred, args.verbose) - else: - print("! Could not find API token in ~/.transifexrc") - return 1 + try: + getter = TransifexGetter() + return get_tx_stats(getter.languages, args.verbose) + except TXError as e: + print("! " + str(e)) + return 1; return 0 if __name__ == "__main__": From a66eabe9efbfad07a2869ab2541fc219916cf7ab Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 31 Jul 2020 12:17:07 +0200 Subject: [PATCH 2/5] i18n: support bogus TX data for testing --- ci/txstats.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ci/txstats.py b/ci/txstats.py index 0f955421e..dbc6fc191 100755 --- a/ci/txstats.py +++ b/ci/txstats.py @@ -54,6 +54,17 @@ class TransifexGetter(object): return None +class BogusGetter(object): + """ + Fake language data. + + This object pretends to retrieve data, and returns fixed language lists and percentages, + for testing purposes without hitting Transifex servers all the time. + """ + def __init__(self): + self.languages = dict() + for lang, completion in ( ("sq", 100), ("ar", 44), ("as", 28), ("de", 15), ("da", 4), ("ts", 82) ): + self.languages[lang] = dict(translated=dict(stringcount=686, percentage=(completion/100.0))) def output_langs(all_langs, label, filterfunc): @@ -125,9 +136,13 @@ def get_tx_stats(languages, verbose): def main(): parser = argparse.ArgumentParser(description="Update Transifex Statistics") parser.add_argument("--verbose", "-v", help="Show statistics", action="store_true") + parser.add_argument("--bogus", "-n", help="Use bogus data (do not query Transifex)", action="store_true") args = parser.parse_args() try: - getter = TransifexGetter() + if args.bogus: + getter = BogusGetter() + else: + getter = TransifexGetter() return get_tx_stats(getter.languages, args.verbose) except TXError as e: print("! " + str(e)) From dacd236f6a2644e76b2c8d380cf812230d61e18a Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 31 Jul 2020 12:22:40 +0200 Subject: [PATCH 3/5] i18n: factor out output method for txstats --- ci/txstats.py | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/ci/txstats.py b/ci/txstats.py index dbc6fc191..23a00bbd5 100755 --- a/ci/txstats.py +++ b/ci/txstats.py @@ -67,7 +67,24 @@ class BogusGetter(object): self.languages[lang] = dict(translated=dict(stringcount=686, percentage=(completion/100.0))) -def output_langs(all_langs, label, filterfunc): +class PrintOutputter(object): + """ + Output via print-statements. + """ + def __init__(self): + pass + + def print(self, s): + print(s) + + def __enter__(self): + return self + + def __exit__(self, e, v, tb): + pass + + +def output_langs(all_langs, outputter, label, filterfunc): """ Output (via print) all of the languages in @p all_langs that satisfy the translation-percentage filter @p filterfunc. @@ -83,13 +100,13 @@ def output_langs(all_langs, label, filterfunc): while len(out) > width - len(prefix): chunk = out[:out[:width].rfind(" ")] - print("%s%s" % (prefix, chunk)) + outputter.print("%s%s" % (prefix, chunk)) out = out[len(chunk)+1:] prefix = " " - print("%s%s" % (prefix, out)) + outputter.print("%s%s" % (prefix, out)) -def get_tx_stats(languages, verbose): +def get_tx_stats(languages, outputter, verbose): """ Does an API request to Transifex with the given API @p token, getting the translation statistics for the main body of texts. Then prints @@ -110,7 +127,7 @@ def get_tx_stats(languages, verbose): ) all_langs = [] - print("# Total %d languages" % len(languages)) + outputter.print("# Total %d languages" % len(languages)) for lang_name in languages: if lang_name in suppressed_languages: continue @@ -124,15 +141,18 @@ def get_tx_stats(languages, verbose): if verbose: for s, l in sorted(all_langs, reverse=True): - print("# %16s\t%6.2f" % (l, s * 100.0)) - output_langs(all_langs, "complete", lambda s : s == 1.0) - output_langs(all_langs, "good", lambda s : 1.0 > s >= 0.75) - output_langs(all_langs, "ok", lambda s : 0.75 > s >= 0.05) - output_langs(all_langs, "incomplete", lambda s : 0.05 > s) + outputter.print("# %16s\t%6.2f" % (l, s * 100.0)) + output_langs(all_langs, outputter, "complete", lambda s : s == 1.0) + output_langs(all_langs, outputter, "good", lambda s : 1.0 > s >= 0.75) + output_langs(all_langs, outputter, "ok", lambda s : 0.75 > s >= 0.05) + output_langs(all_langs, outputter, "incomplete", lambda s : 0.05 > s) return 0 +def get_outputter(): + return PrintOutputter() + def main(): parser = argparse.ArgumentParser(description="Update Transifex Statistics") parser.add_argument("--verbose", "-v", help="Show statistics", action="store_true") @@ -143,7 +163,8 @@ def main(): getter = BogusGetter() else: getter = TransifexGetter() - return get_tx_stats(getter.languages, args.verbose) + with get_outputter() as outputter: + return get_tx_stats(getter.languages, outputter, args.verbose) except TXError as e: print("! " + str(e)) return 1; From 1a87879f9bdbf478753c03723df25e5f75598216 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 31 Jul 2020 12:43:52 +0200 Subject: [PATCH 4/5] i18n: enable updating stats-in-place --- ci/txstats.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/ci/txstats.py b/ci/txstats.py index 23a00bbd5..85b3896fe 100755 --- a/ci/txstats.py +++ b/ci/txstats.py @@ -84,6 +84,56 @@ class PrintOutputter(object): pass +class EditingOutputter(object): + """ + Edit CMakeLists in-place. + """ + def __init__(self): + with open("CMakeLists.txt", "r") as f: + lines = f.readlines() + + mark = None + for l in lines: + # Note that we didn't strip the lines, so need the \n here + if l.startswith("# Total ") and l.endswith(" languages\n"): + mark = lines.index(l) + break + if mark is None: + raise TXError("No CMakeLists.txt lines for TX stats found") + self.pre_lines = lines[:mark] + + nextmark = mark + 1 + for l in lines[mark+1:]: + if l.startswith("set( _tx_"): + nextmark += 1 + continue + if l.startswith(" "): + nextmark += 1 + continue + break + if nextmark > mark + 12 or nextmark > len(lines) - 4: + # Try to catch runaway nextmarks: we know there should + # be four set-lines, which are unlikely to be 3 lines each; + # similarly the CMakeLists.txt is supposed to end with + # some boilerplate. + raise TXError("Could not find end of TX settings in CMakeLists.txt") + self.post_lines = lines[nextmark:] + + self.mid_lines = [] + + def print(self, s): + # Add the implicit \n from print() + self.mid_lines.append(s + "\n") + + def __enter__(self): + return self + + def __exit__(self, e, v, tb): + if e is None: + with open("CMakeLists.txt", "w") as f: + f.write("".join(self.pre_lines + self.mid_lines + self.post_lines)) + + def output_langs(all_langs, outputter, label, filterfunc): """ Output (via print) all of the languages in @p all_langs @@ -150,20 +200,22 @@ def get_tx_stats(languages, outputter, verbose): return 0 -def get_outputter(): - return PrintOutputter() - def main(): parser = argparse.ArgumentParser(description="Update Transifex Statistics") parser.add_argument("--verbose", "-v", help="Show statistics", action="store_true") parser.add_argument("--bogus", "-n", help="Use bogus data (do not query Transifex)", action="store_true") + parser.add_argument("--edit", "-e", help="Edit CMakeLists.txt in-place", action="store_true") args = parser.parse_args() try: if args.bogus: getter = BogusGetter() else: getter = TransifexGetter() - with get_outputter() as outputter: + if args.edit: + outputter = EditingOutputter() + else: + outputter = PrintOutputter() + with outputter: return get_tx_stats(getter.languages, outputter, args.verbose) except TXError as e: print("! " + str(e)) From 5380f8062d7c7286cf665da97dd11099eddd84ef Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 31 Jul 2020 12:46:52 +0200 Subject: [PATCH 5/5] i18n: when editing CMakeLists in-place, be a little more verbose --- ci/txstats.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/txstats.py b/ci/txstats.py index 85b3896fe..a9f412743 100755 --- a/ci/txstats.py +++ b/ci/txstats.py @@ -120,10 +120,13 @@ class EditingOutputter(object): self.post_lines = lines[nextmark:] self.mid_lines = [] + print("# Editing CMakeLists.txt in-place") def print(self, s): # Add the implicit \n from print() self.mid_lines.append(s + "\n") + if s.startswith("#"): + print(s) def __enter__(self): return self @@ -132,6 +135,7 @@ class EditingOutputter(object): if e is None: with open("CMakeLists.txt", "w") as f: f.write("".join(self.pre_lines + self.mid_lines + self.post_lines)) + print("# CMakeLists.txt updated") def output_langs(all_langs, outputter, label, filterfunc):