could you please give me some advice on how to read directory or a lot of tar, thanks!!!

[leo23ui]

Hello, I converted the yfcc15m dataset into multiple tar files, each containing 1000 data points.
But - "train" -data/home/gg/gg/MQBench - the main/test/model/tran1 / batch_1 tar \ can only read a tar to do.
since " --train-data /home/gg/gg/MQBench-main/test/model/tran1/*.tar " and " --train-data /home/gg/gg/MQBench-main/test/model/tran1" will report error,
could you please give me some advice on how to read directory or a lot of tar, thanks!!!

[leo23ui]

could i use the following function to load directory which contains many tars ,thanks!!

def get_wds_dataset(args, preprocess_img, is_train, epoch=0, floor=False, tokenizer=None):

def get_tar_files_from_folder(folder):
    return sorted(glob.glob(os.path.join(folder, "*.tar")))


if is_train:
    if os.path.isdir(args.train_data): 
        input_shards = get_tar_files_from_folder(args.train_data)
    else:
        input_shards = args.train_data  
else:
    if os.path.isdir(args.val_data):  
        input_shards = get_tar_files_from_folder(args.val_data)
    else:
        input_shards = args.val_data
print(f"input_shards type: {type(input_shards)}")


if isinstance(input_shards, list):
    print(f"input_shards number: {len(input_shards)}")
else:
    print("input_shards number: 1")


assert input_shards is not None
resampled = getattr(args, 'dataset_resampled', False) and is_train

num_shards = None
if is_train:
    if args.train_num_samples is not None:
        num_samples = args.train_num_samples
    else:
        num_samples, num_shards = get_dataset_size(input_shards)
        if not num_samples:
            raise RuntimeError(
                'Currently, the number of dataset samples must be specified for the training dataset. '
                'Please specify it via `--train-num-samples` if no dataset length info is present.')
else:
    num_samples = args.val_num_samples or 0


shared_epoch = SharedEpoch(epoch=epoch)


pipeline = [wds.SimpleShardList(input_shards)]


if is_train:
    if not resampled:
        pipeline.extend([
            detshuffle2(
                bufsize=_SHARD_SHUFFLE_SIZE,
                initial=_SHARD_SHUFFLE_INITIAL,
                seed=args.seed,
                epoch=shared_epoch,
            ),
            wds.split_by_node,
            wds.split_by_worker,
        ])
    pipeline.extend([
        tarfile_to_samples_nothrow,
        wds.shuffle(
            bufsize=_SAMPLE_SHUFFLE_SIZE,
            initial=_SAMPLE_SHUFFLE_INITIAL,
        ),
    ])
else:
    pipeline.extend([
        wds.split_by_worker,
        wds.tarfile_to_samples(handler=log_and_continue),
    ])
pipeline.extend([
    wds.select(filter_no_caption_or_no_image),
    wds.decode("pilrgb", handler=log_and_continue),
    wds.rename(image="jpg;png;jpeg;webp", text="txt"),
    wds.map_dict(image=preprocess_img,
                 text=lambda text: tokenizer(text)[0]),
    wds.to_tuple("image", "text"),
    wds.batched(args.batch_size, partial=not is_train)
])

dataset = wds.DataPipeline(*pipeline)


if is_train:
    if not resampled:
        num_shards = num_shards or len(input_shards)
        assert num_shards >= args.workers * args.world_size, 'number of shards must be >= total workers'
    round_fn = math.floor if floor else math.ceil
    global_batch_size = args.batch_size * args.world_size
    num_batches = round_fn(num_samples / global_batch_size)
    num_workers = max(1, args.workers)
    num_worker_batches = round_fn(num_batches / num_workers)
    num_batches = num_worker_batches * num_workers
    num_samples = num_batches * global_batch_size
    dataset = dataset.with_epoch(num_worker_batches)
else:
    num_batches = math.ceil(num_samples / args.batch_size)

dataloader = wds.WebLoader(
    dataset,
    batch_size=None,
    shuffle=False,
    num_workers=args.workers,
    persistent_workers=args.workers > 0,
)


dataloader.num_batches = num_batches
dataloader.num_samples = num_samples

return DataInfo(dataloader=dataloader, shared_epoch=shared_epoch)