[CLS] question [SEP] context [SEP]

context = raw_datasets["train"][0]["context"]

question = raw_datasets["train"][0]["question"]

inputs = tokenizer(question, context)

tokenizer.decode(inputs["input_ids"])

'[CLS] To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France? [SEP] Architecturally, '

'the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin '

'Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms '

'upraised with the legend " Venite Ad Me Omnes ". Next to the Main Building is the Basilica of the Sacred '

'Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a '

'replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette '

'Soubirous in 1858. At the end of the main drive ( and in a direct line that connects through 3 statues '

'and the Gold Dome ), is a simple, modern stone statue of Mary. [SEP]'

inputs = tokenizer(

    question,

    context,

    max_length=100,

    truncation="only_second",

    stride=50,

    return_overflowing_tokens=True,

)

inputs.keys()

# dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'overflow_to_sample_mapping'])

for ids in inputs["input_ids"]:

    print(tokenizer.decode(ids))

'[CLS] To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France? [SEP] Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend " Venite Ad Me Omnes ". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basi [SEP]'

'[CLS] To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France? [SEP] the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend " Venite Ad Me Omnes ". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin [SEP]'

'[CLS] To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France? [SEP] Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive ( and in a direct line that connects through 3 [SEP]'

'[CLS] To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France? [SEP]. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive ( and in a direct line that connects through 3 statues and the Gold Dome ), is a simple, modern stone statue of Mary. [SEP]'

inputs = tokenizer(

    question,

    context,

    max_length=100,

    truncation="only_second",

    stride=50,

    return_overflowing_tokens=True,

    return_offsets_mapping=True,

)

inputs.keys()

dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'offset_mapping', 'overflow_to_sample_mapping'])

inputs["overflow_to_sample_mapping"]

[0, 0, 0, 0]

inputs = tokenizer(

    raw_datasets["train"][2:6]["question"],

    raw_datasets["train"][2:6]["context"],

    max_length=100,

    truncation="only_second",

    stride=50,

    return_overflowing_tokens=True,

    return_offsets_mapping=True,

)

print(f"The 4 examples gave {len(inputs['input_ids'])} features.")

print(f"Here is where each comes from: {inputs['overflow_to_sample_mapping']}.")

'The 4 examples gave 19 features.'

'Here is where each comes from: [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3].'

from datasets import load_dataset

raw_datasets = load_dataset("squad")

raw_datasets["train"].filter(lambda x: len(x["answers"]["text"]) != 1)

# 处理训练数据

from transformers import AutoTokenizer

model_checkpoint = "bert-base-cased"

tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)

max_length = 384

stride = 128

def preprocess_training_examples(examples):

    questions = [q.strip() for q in examples["question"]]

    inputs = tokenizer(

        questions,

        examples["context"],

        max_length=max_length,

        truncation="only_second",

        stride=stride,

        return_overflowing_tokens=True,

        return_offsets_mapping=True,

        padding="max_length",

    )

    offset_mapping = inputs.pop("offset_mapping")

    sample_map = inputs.pop("overflow_to_sample_mapping")

    answers = examples["answers"]

    start_positions = []

    end_positions = []

    for i, offset in enumerate(offset_mapping):

        sample_idx = sample_map[i]

        answer = answers[sample_idx]

        start_char = answer["answer_start"][0]

        end_char = answer["answer_start"][0] + len(answer["text"][0])

        sequence_ids = inputs.sequence_ids(i)

        # Find the start and end of the context

        idx = 0

        while sequence_ids[idx] != 1:

            idx += 1

        context_start = idx

        while sequence_ids[idx] == 1:

            idx += 1

        context_end = idx - 1

        # If the answer is not fully inside the context, label is (0, 0)

        if offset[context_start][0] > start_char or offset[context_end][1] < end_char:

            start_positions.append(0)

            end_positions.append(0)

        else:

            # Otherwise it's the start and end token positions

            idx = context_start

            while idx <= context_end and offset[idx][0] <= start_char:

                idx += 1

            start_positions.append(idx - 1)

            idx = context_end

            while idx >= context_start and offset[idx][1] >= end_char:

                idx -= 1

            end_positions.append(idx + 1)

    inputs["start_positions"] = start_positions

    inputs["end_positions"] = end_positions

    return inputs

train_dataset = raw_datasets["train"].map(

    preprocess_training_examples,

    batched=True,

    remove_columns=raw_datasets["train"].column_names,

)

# 处理验证数据

def preprocess_validation_examples(examples):

    questions = [q.strip() for q in examples["question"]]

    inputs = tokenizer(

        questions,

        examples["context"],

        max_length=max_length,

        truncation="only_second",

        stride=stride,

        return_overflowing_tokens=True,

        return_offsets_mapping=True,

        padding="max_length",

    )

    sample_map = inputs.pop("overflow_to_sample_mapping")

    example_ids = []

    for i in range(len(inputs["input_ids"])):

        sample_idx = sample_map[i]

        example_ids.append(examples["id"][sample_idx])

        sequence_ids = inputs.sequence_ids(i)

        offset = inputs["offset_mapping"][i]

        inputs["offset_mapping"][i] = [

            o if sequence_ids[k] == 1 else None for k, o in enumerate(offset)

        ]

    inputs["example_id"] = example_ids

    return inputs

validation_dataset = raw_datasets["validation"].map(

    preprocess_validation_examples,

    batched=True,

    remove_columns=raw_datasets["validation"].column_names,

)

from datasets import load_metric

metric = load_metric("squad")

from tqdm.auto import tqdm

def compute_metrics(start_logits, end_logits, features, examples):

    example_to_features = collections.defaultdict(list)

    for idx, feature in enumerate(features):

        example_to_features[feature["example_id"]].append(idx)

    predicted_answers = []

    for example in tqdm(examples):

        example_id = example["id"]

        context = example["context"]

        answers = []

        # Loop through all features associated with that example

        for feature_index in example_to_features[example_id]:

            start_logit = start_logits[feature_index]

            end_logit = end_logits[feature_index]

            offsets = features[feature_index]["offset_mapping"]

            start_indexes = np.argsort(start_logit)[-1 : -n_best - 1 : -1].tolist()

            end_indexes = np.argsort(end_logit)[-1 : -n_best - 1 : -1].tolist()

            for start_index in start_indexes:

                for end_index in end_indexes:

                    # Skip answers that are not fully in the context

                    if offsets[start_index] is None or offsets[end_index] is None:

                        continue

                    # Skip answers with a length that is either < 0 or > max_answer_length

                    if (

                        end_index < start_index

                        or end_index - start_index + 1 > max_answer_length

                    ):

                        continue

                    answer = {

                        "text": context[offsets[start_index][0] : offsets[end_index][1]],

                        "logit_score": start_logit[start_index] + end_logit[end_index],

                    }

                    answers.append(answer)

        # Select the answer with the best score

        if len(answers) > 0:

            best_answer = max(answers, key=lambda x: x["logit_score"])

            predicted_answers.append(

                {"id": example_id, "prediction_text": best_answer["text"]}

            )

        else:

            predicted_answers.append({"id": example_id, "prediction_text": ""})

    theoretical_answers = [{"id": ex["id"], "answers": ex["answers"]} for ex in examples]

    return metric.compute(predictions=predicted_answers, references=theoretical_answers)

model = AutoModelForQuestionAnswering.from_pretrained(model_checkpoint)

from transformers import TrainingArguments

args = TrainingArguments(

    "bert-finetuned-squad",

    evaluation_strategy="no",

    save_strategy="epoch",

    learning_rate=2e-5,

    num_train_epochs=3,

    weight_decay=0.01,

    fp16=True,

    push_to_hub=True,

)

from transformers import Trainer

trainer = Trainer(

    model=model,

    args=args,

    train_dataset=train_dataset,

    eval_dataset=validation_dataset,

    tokenizer=tokenizer,

)

trainer.train()