convert : fix TypeError on GPT-2 vocab.json (ggml-org#5288)

Sang-Kil Park · web-flow · commit f68664ac241a · 2024-02-06T23:28:00.000-05:00
diff --git a/convert.py b/convert.py
@@ -334,9 +334,9 @@ def load(model_plus: ModelPlus) -> Params:
 class BpeVocab:
     def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None:
         self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read())
-        try:
+        if isinstance(self.bpe_tokenizer.get('model'), dict):
             self.vocab = self.bpe_tokenizer["model"]["vocab"]
-        except KeyError:
+        else:
             self.vocab = self.bpe_tokenizer
         added_tokens: dict[str, int]
         if fname_added_tokens is not None:
