From 7b536215091edec882cfae9dc2530034260d6f98 Mon Sep 17 00:00:00 2001 From: webbrain-one <295484252+webbrain-one@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:54:19 +0300 Subject: [PATCH] Set max input length to 256 for BERT tokenization WordPiece tokenization can produce inputs longer than 128 tokens, so increase the max length in run.py. --- run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run.py b/run.py index cbb1d48..8049b7c 100644 --- a/run.py +++ b/run.py @@ -107,7 +107,7 @@ def eval(iter_data, model): parser.add_argument("--batch_size", type=int, default=32) parser.add_argument("--lr", type=float, default=3e-5) parser.add_argument("--n_epochs", type=int, default=5) - parser.add_argument("--max_len", type=int, default=128) + parser.add_argument("--max_len", type=int, default=256) parser.add_argument("--pretrained_model_name", type=str, default="bert-base-cased") parser.add_argument("--train", dest="train", action="store_true") parser.add_argument("--existing_model_path", type=str, default=None) @@ -131,13 +131,13 @@ def eval(iter_data, model): label_map[label] = i train_dataset = NERDataSet(data_list=train_examples, tokenizer=tokenizer, label_map=label_map, - max_len=128) + max_len=args.max_len) eval_dataset = NERDataSet(data_list=val_examples, tokenizer=tokenizer, label_map=label_map, - max_len=128) + max_len=args.max_len) test_dataset = NERDataSet(data_list=test_examples, tokenizer=tokenizer, label_map=label_map, - max_len=128) + max_len=args.max_len) train_iter = data.DataLoader(dataset=train_dataset, batch_size=args.batch_size,