Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support torch model bin format #33

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Support torch model bin format #33

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
from bitsandbytes.nn import Linear4bit, Params4bit
from accelerate import init_empty_weights
from accelerate.utils import set_seed
from transformers.utils import hub, SAFE_WEIGHTS_NAME, SAFE_WEIGHTS_INDEX_NAME
from transformers.utils import hub, SAFE_WEIGHTS_NAME, SAFE_WEIGHTS_INDEX_NAME, WEIGHTS_NAME, WEIGHTS_INDEX_NAME
from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
from transformers.tokenization_utils_fast import PreTrainedTokenizerFast
from fastcore.parallel import parallel
Expand Down Expand Up @@ -595,13 +595,20 @@ def fsdp_main(local_rank:int, world_size:int, args:Dict):

# Grab the safetensors files that hold the weights
try:
idx = hub.cached_file(args["model_name"], SAFE_WEIGHTS_INDEX_NAME)
try:
idx = hub.cached_file(args["model_name"], SAFE_WEIGHTS_INDEX_NAME)
except OSError:
idx = hub.cached_file(args["model_name"], WEIGHTS_INDEX_NAME)
files, _ = hub.get_checkpoint_shard_files(args["model_name"], idx)
except OSError:
try:
# This means the model doesn't have a model.safetensors.index.json because it is not sharded
files = []
files.append(hub.cached_file(args["model_name"], SAFE_WEIGHTS_NAME))
try:
model_file = hub.cached_file(args["model_name"], SAFE_WEIGHTS_NAME)
except OSError:
model_file = hub.cached_file(args["model_name"], WEIGHTS_NAME)
files.append(model_file)
except OSError as e:
# This means the model probably doesn't have a safetensors file
raise e
Expand All @@ -624,7 +631,10 @@ def load_and_quantize_parallel(name_param, model, **kwargs):

start = time.time()
for filename in files:
weights = safetensors.torch.load_file(filename)
if filename.endswith(".safetensors"):
weights = safetensors.torch.load_file(filename)
else:
weights = torch.load(filename)
parallel(load_and_quantize_parallel, iter(weights.items()), n_workers=n_workers, threadpool=True,
model=model, dtype=torch_dtype, device=local_rank, skip_names=load_param_skip_names,
to_cpu=(args["low_memory"] and rank==0), to_meta=(args["low_memory"] and rank!=0),
Expand Down Expand Up @@ -1018,4 +1028,4 @@ def main(
mp.spawn(fsdp_main,
args=(world_size, args),
nprocs=torch.cuda.device_count(),
join=True)
join=True)