Skip to content

Commit

Permalink
cosmetics
Browse files Browse the repository at this point in the history
  • Loading branch information
rasbt authored May 5, 2024
1 parent e9bdbf0 commit 2e9d5ac
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions ch06/01_main-chapter-code/previous_chapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -312,10 +312,10 @@ def load_weights_into_gpt(gpt, params):

def text_to_token_ids(text, tokenizer):
encoded = tokenizer.encode(text, allowed_special={'<|endoftext|>'})
encoded_tensor = torch.tensor(encoded).unsqueeze(0) # add batch dimension
encoded_tensor = torch.tensor(encoded).unsqueeze(0) # add batch dimension
return encoded_tensor


def token_ids_to_text(token_ids, tokenizer):
flat = token_ids.squeeze(0) # remove batch dimension
flat = token_ids.squeeze(0) # remove batch dimension
return tokenizer.decode(flat.tolist())

0 comments on commit 2e9d5ac

Please sign in to comment.