score.py

import conlangs.conlangs as conlangs
import jsonlines

from absl import app
from absl import flags
from Levenshtein import distance

TEST = flags.DEFINE_string("test", None, "Path to test file")
VERBOSE = flags.DEFINE_bool("verbose", False, "Show errors")

GRAMMARS = {
  "ferulian": conlangs.CARDINALS_FERULIAN,
  "neoferulian": conlangs.CARDINALS_NEOFERULIAN,
  "archiferulian": conlangs.CARDINALS_ARCHIFERULIAN,
  "continental_ferulian": conlangs.CARDINALS_CONTINENTAL_FERULIAN,
  "lurefian": conlangs.CARDINALS_LUREFIAN,
  "archilurefian": conlangs.CARDINALS_ARCHILUREFIAN,
}


def score_output(path, out):
  language = path.split("/")[-1].split(".")[0]
  print(f"Scoring {language}")
  grammar = GRAMMARS[language]
  total_names = 0
  total_words = 0
  total_chars = 0
  err_names = 0
  err_words = 0
  err_chars = 0
  with jsonlines.open(path) as reader:
    for elt in reader:
      hyp = elt["reading"].strip()
      ref = grammar.verbalize(elt["number"])[0]
      total_names += 1
      total_words += len(ref.split())
      total_chars += len(ref)
      if hyp != ref:
        err_names += 1
        err_words += distance(ref.split(), hyp.split())
        err_chars += distance(ref, hyp)
        if VERBOSE.value:
          out.write("-" * 80 + "\n")
          out.write(f"Num:\t{elt['number']}\n")
          out.write(f"Ref:\t{ref}\n")
          out.write(f"Hyp:\t{hyp}\n")
  out.write("*" * 80 + "\n")
  out.write(f"______\tTOTAL\tERROR\tERROR RATE\n")
  out.write(
    f"Names:\t{total_names}\t{err_names}\t{err_names / total_names:.02f}\n"
  )
  out.write(
    f"Words:\t{total_words}\t{err_words}\t{err_words / total_words:.02f}\n"
  )
  out.write(
    f"Chars:\t{total_chars}\t{err_chars}\t{err_chars / total_chars:.02f}\n"
  )


def main(unused_argv):
  output_file = TEST.value.replace(".jsonl", ".txt")
  with open(output_file, "w") as out:
    score_output(TEST.value, out)


if __name__ == "__main__":
  flags.mark_flag_as_required("test")
  app.run(main)