When using a Hugging Face dataset on a runpod instance, I get 429 errors that are not effectively rate limited and ultimately cause an exception after 5/5 retries. In the logs, retries seem to appear immediately and do not back off to abide by the HF requested rate limit. Also request ability to manually set max_workers to a reasonable number in the UI or limit it to something like 16, in case the default max_workers is set to the number of CPU threads, which can be significant for some servers.
This could also be a bug in Hugging Face not backing off when hitting the rate limit.
I expected hf download to work correctly. Manually downloading on the remote server with hf download --num-workers 16 works as expected and backs off when rate limited without crashing.
Traceback (most recent call last):
File "/workspace/OneTrainer/scripts/train_remote.py", line 102, in <module>
main()
File "/workspace/OneTrainer/scripts/train_remote.py", line 89, in main
trainer.train()
File "/workspace/OneTrainer/modules/trainer/GenericTrainer.py", line 646, in train
self.data_loader.get_data_set().start_next_epoch()
File "/workspace/OneTrainer/venv/src/mgds/src/mgds/MGDS.py", line 49, in start_next_epoch
self.loading_pipeline.start_next_epoch()
File "/workspace/OneTrainer/venv/src/mgds/src/mgds/LoadingPipeline.py", line 93, in start_next_epoch
module.start(self.__current_epoch)
File "/workspace/OneTrainer/venv/src/mgds/src/mgds/pipelineModules/DownloadHuggingfaceDatasets.py", line 49, in start
hf_path = huggingface_hub.snapshot_download(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/_snapshot_download.py", line 332, in snapshot_download
thread_map(
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/tqdm/contrib/concurrent.py", line 69, in thread_map
return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/tqdm/contrib/concurrent.py", line 51, in _executor_map
return list(tqdm_class(ex.map(fn, *iterables, chunksize=chunksize), **kwargs))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/tqdm/std.py", line 1181, in __iter__
for obj in iterable:
File "/usr/lib/python3.12/concurrent/futures/_base.py", line 619, in result_iterator
yield _result_or_cancel(fs.pop())
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/concurrent/futures/_base.py", line 317, in _result_or_cancel
return fut.result(timeout)
^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/concurrent/futures/_base.py", line 456, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/_snapshot_download.py", line 306, in _inner_hf_hub_download
return hf_hub_download(
^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1010, in hf_hub_download
return _hf_hub_download_to_cache_dir(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1171, in _hf_hub_download_to_cache_dir
_download_to_tmp_and_move(
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1738, in _download_to_tmp_and_move
http_get(
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 426, in http_get
r = _request_wrapper(
^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 309, in _request_wrapper
response = http_backoff(method=method, url=url, **params, retry_on_exceptions=(), retry_on_status_codes=(429,))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 317, in http_backoff
response.raise_for_status() # Will raise uncaught exception
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/OneTrainer/venv/lib/python3.12/site-packages/requests/models.py", line 1028, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 429 Client Error: Too Many Requests for url: https://huggingface.co/datasets/cornbox/chroma_finetune_dataset/resolve/fab23bb751ea257e1c4899c926167fa9f6218e0b/clothes__belly_chain/93812564_029_7dc9.txt
Traceback (most recent call last):
File "/mnt/linux_extra/projects/OneTrainer/modules/ui/TrainUI.py", line 719, in __training_thread_function
trainer.train()
File "/mnt/linux_extra/projects/OneTrainer/modules/trainer/CloudTrainer.py", line 99, in train
self.cloud.run_trainer()
File "/mnt/linux_extra/projects/OneTrainer/modules/cloud/LinuxCloud.py", line 176, in run_trainer
self.__trail_detached_trainer()
File "/mnt/linux_extra/projects/OneTrainer/modules/cloud/LinuxCloud.py", line 186, in __trail_detached_trainer
self.connection.run(f'exit $(<{self.exit_status_file})',in_stream=False)
File "/mnt/linux_extra/projects/OneTrainer/conda_env/lib/python3.10/site-packages/decorator.py", line 235, in fun
return caller(func, *(extras + args), **kw)
File "/mnt/linux_extra/projects/OneTrainer/conda_env/lib/python3.10/site-packages/fabric/connection.py", line 23, in opens
return method(self, *args, **kwargs)
File "/mnt/linux_extra/projects/OneTrainer/conda_env/lib/python3.10/site-packages/fabric/connection.py", line 763, in run
return self._run(self._remote_runner(), command, **kwargs)
File "/mnt/linux_extra/projects/OneTrainer/conda_env/lib/python3.10/site-packages/invoke/context.py", line 129, in _run
return runner.run(command, **kwargs)
File "/mnt/linux_extra/projects/OneTrainer/conda_env/lib/python3.10/site-packages/fabric/runners.py", line 83, in run
return super().run(command, **kwargs)
File "/mnt/linux_extra/projects/OneTrainer/conda_env/lib/python3.10/site-packages/invoke/runners.py", line 402, in run
return self._run_body(command, **kwargs)
File "/mnt/linux_extra/projects/OneTrainer/conda_env/lib/python3.10/site-packages/invoke/runners.py", line 468, in _run_body
return self.make_promise() if self._asynchronous else self._finish()
File "/mnt/linux_extra/projects/OneTrainer/conda_env/lib/python3.10/site-packages/invoke/runners.py", line 535, in _finish
raise UnexpectedExit(result)
invoke.exceptions.UnexpectedExit: Encountered a bad command exit code!
What happened?
When using a Hugging Face dataset on a runpod instance, I get 429 errors that are not effectively rate limited and ultimately cause an exception after 5/5 retries. In the logs, retries seem to appear immediately and do not back off to abide by the HF requested rate limit. Also request ability to manually set max_workers to a reasonable number in the UI or limit it to something like 16, in case the default max_workers is set to the number of CPU threads, which can be significant for some servers.
This could also be a bug in Hugging Face not backing off when hitting the rate limit.
What did you expect would happen?
I expected hf download to work correctly. Manually downloading on the remote server with hf download --num-workers 16 works as expected and backs off when rate limited without crashing.
Relevant log output
Generate and upload debug_report.log
No response