libarchive/test/test_tar_large.c - third_party/libarchive - Git at Google

 /*-
  * Copyright (c) 2003-2007 Tim Kientzle
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "test.h"
 __FBSDID("$FreeBSD: head/lib/libarchive/test/test_tar_large.c 201247 2009-12-30 05:59:21Z kientzle $");

 #include <errno.h>
 #include <stdlib.h>
 #include <string.h>

 /*
  * This is a somewhat tricky test that verifies the ability to
  * write and read very large entries to tar archives.  It
  * writes entries from 2GB up to 1TB to an archive in memory.
  * The memory storage here carefully avoids actually storing
  * any part of the file bodies, so it runs very quickly and requires
  * very little memory.  If you're willing to wait a few minutes,
  * you should be able to exercise petabyte entries with this code.
  */

 /*
  * Each file is built up by duplicating the following block.
  */
 static size_t filedatasize;
 static void *filedata;

 /*
  * We store the archive as blocks of data generated by libarchive,
  * each possibly followed by bytes of file data.
  */
 struct memblock {
 	struct memblock *next;
 	size_t	size;
 	void *buff;
 	int64_t filebytes;
 };

 /*
  * The total memory store is just a list of memblocks plus
  * some accounting overhead.
  */
 struct memdata {
 	int64_t filebytes;
 	void *buff;
 	struct memblock *first;
 	struct memblock *last;
 };

 /* The following size definitions simplify things below. */
 #define KB ((int64_t)1024)
 #define MB ((int64_t)1024 * KB)
 #define GB ((int64_t)1024 * MB)
 #define TB ((int64_t)1024 * GB)

 static int64_t	memory_read_skip(struct archive *, void *, int64_t request);
 static ssize_t	memory_read(struct archive *, void *, const void **buff);
 static ssize_t	memory_write(struct archive *, void *, const void *, size_t);


 static ssize_t
 memory_write(struct archive *a, void *_private, const void *buff, size_t size)
 {
 	struct memdata *private = _private;
 	struct memblock *block;

 	(void)a;

 	/*
 	 * Since libarchive tries to behave in a zero-copy manner, if
 	 * you give a pointer to filedata to the library, a pointer
 	 * into that data will (usually) pop out here.  This way, we
 	 * can tell the difference between filedata and library header
 	 * and metadata.
 	 */
 	if ((const char *)filedata <= (const char *)buff
 	    && (const char *)buff < (const char *)filedata + filedatasize) {
 		/* We don't need to store a block of file data. */
 		private->last->filebytes += (int64_t)size;
 	} else {
 		/* Yes, we're assuming the very first write is metadata. */
 		/* It's header or metadata, copy and save it. */
 		block = (struct memblock *)malloc(sizeof(*block));
 		memset(block, 0, sizeof(*block));
 		block->size = size;
 		block->buff = malloc(size);
 		memcpy(block->buff, buff, size);
 		if (private->last == NULL) {
 			private->first = private->last = block;
 		} else {
 			private->last->next = block;
 			private->last = block;
 		}
 		block->next = NULL;
 	}
 	return ((long)size);
 }

 static ssize_t
 memory_read(struct archive *a, void *_private, const void **buff)
 {
 	struct memdata *private = _private;
 	struct memblock *block;
 	ssize_t size;

 	(void)a;

 	free(private->buff);
 	private->buff = NULL;
 	if (private->first == NULL) {
 		private->last = NULL;
 		return (ARCHIVE_EOF);
 	}
 	if (private->filebytes > 0) {
 		/*
 		 * We're returning file bytes, simulate it by
 		 * passing blocks from the template data.
 		 */
 		if (private->filebytes > (int64_t)filedatasize)
 			size = (ssize_t)filedatasize;
 		else
 			size = (ssize_t)private->filebytes;
 		private->filebytes -= size;
 		*buff = filedata;
 	} else {
 		/*
 		 * We need to get some real data to return.
 		 */
 		block = private->first;
 		private->first = block->next;
 		size = (ssize_t)block->size;
 		if (block->buff != NULL) {
 			private->buff = block->buff;
 			*buff = block->buff;
 		} else {
 			private->buff = NULL;
 			*buff = filedata;
 		}
 		private->filebytes = block->filebytes;
 		free(block);
 	}
 	return (size);
 }


 static int64_t
 memory_read_skip(struct archive *a, void *_private, int64_t skip)
 {
 	struct memdata *private = _private;

 	(void)a;

 	if (private->first == NULL) {
 		private->last = NULL;
 		return (0);
 	}
 	if (private->filebytes > 0) {
 		if (private->filebytes < skip)
 			skip = (off_t)private->filebytes;
 		private->filebytes -= skip;
 	} else {
 		skip = 0;
 	}
 	return (skip);
 }

 DEFINE_TEST(test_tar_large)
 {
 	/* The sizes of the entries we're going to generate. */
 	static int64_t tests[] = {
 		/* Test for 32-bit signed overflow. */
 		2 * GB - 1, 2 * GB, 2 * GB + 1,
 		/* Test for 32-bit unsigned overflow. */
 		4 * GB - 1, 4 * GB, 4 * GB + 1,
 		/* 8GB is the "official" max for ustar. */
 		8 * GB - 1, 8 * GB, 8 * GB + 1,
 		/* Bend ustar a tad and you can get 64GB (12 octal digits). */
 		64 * GB - 1, 64 * GB,
 		/* And larger entries that require non-ustar extensions. */
 		256 * GB, 1 * TB, 0 };
 	int i;
 	char namebuff[64];
 	struct memdata memdata;
 	struct archive_entry *ae;
 	struct archive *a;
 	int64_t  filesize;
 	size_t writesize;

 	filedatasize = (size_t)(1 * MB);
 	filedata = malloc(filedatasize);
 	memset(filedata, 0xAA, filedatasize);
 	memset(&memdata, 0, sizeof(memdata));

 	/*
 	 * Open an archive for writing.
 	 */
 	a = archive_write_new();
 	archive_write_set_format_pax_restricted(a);
 	archive_write_set_bytes_per_block(a, 0); /* No buffering. */
 	archive_write_open(a, &memdata, NULL, memory_write, NULL);

 	/*
 	 * Write a series of large files to it.
 	 */
 	for (i = 0; tests[i] != 0; i++) {
 		assert((ae = archive_entry_new()) != NULL);
 		sprintf(namebuff, "file_%d", i);
 		archive_entry_copy_pathname(ae, namebuff);
 		archive_entry_set_mode(ae, S_IFREG | 0755);
 		filesize = tests[i];

 		archive_entry_set_size(ae, filesize);

 		assertA(0 == archive_write_header(a, ae));
 		archive_entry_free(ae);

 		/*
 		 * Write the actual data to the archive.
 		 */
 		while (filesize > 0) {
 			writesize = filedatasize;
 			if ((int64_t)writesize > filesize)
 				writesize = (size_t)filesize;
 			assertA((int)writesize
 			    == archive_write_data(a, filedata, writesize));
 			filesize -= writesize;
 		}
 	}

 	assert((ae = archive_entry_new()) != NULL);
 	archive_entry_copy_pathname(ae, "lastfile");
 	archive_entry_set_mode(ae, S_IFREG | 0755);
 	assertA(0 == archive_write_header(a, ae));
 	archive_entry_free(ae);


 	/* Close out the archive. */
 	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
 	assertEqualInt(ARCHIVE_OK, archive_write_free(a));

 	/*
 	 * Open the same archive for reading.
 	 */
 	a = archive_read_new();
 	archive_read_support_format_tar(a);
 	archive_read_open2(a, &memdata, NULL,
 	    memory_read, memory_read_skip, NULL);

 	/*
 	 * Read entries back.
 	 */
 	for (i = 0; tests[i] > 0; i++) {
 		assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
 		sprintf(namebuff, "file_%d", i);
 		assertEqualString(namebuff, archive_entry_pathname(ae));
 		assert(tests[i] == archive_entry_size(ae));
 	}
 	assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
 	assertEqualString("lastfile", archive_entry_pathname(ae));

 	assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));

 	/* Close out the archive. */
 	assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
 	assertEqualInt(ARCHIVE_OK, archive_read_free(a));

 	free(memdata.buff);
 	free(filedata);
 }
	/*-
	* Copyright (c) 2003-2007 Tim Kientzle
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/
	#include "test.h"
	__FBSDID("$FreeBSD: head/lib/libarchive/test/test_tar_large.c 201247 2009-12-30 05:59:21Z kientzle $");

	#include <errno.h>
	#include <stdlib.h>
	#include <string.h>

	/*
	* This is a somewhat tricky test that verifies the ability to
	* write and read very large entries to tar archives. It
	* writes entries from 2GB up to 1TB to an archive in memory.
	* The memory storage here carefully avoids actually storing
	* any part of the file bodies, so it runs very quickly and requires
	* very little memory. If you're willing to wait a few minutes,
	* you should be able to exercise petabyte entries with this code.
	*/

	/*
	* Each file is built up by duplicating the following block.
	*/
	static size_t filedatasize;
	static void *filedata;

	/*
	* We store the archive as blocks of data generated by libarchive,
	* each possibly followed by bytes of file data.
	*/
	struct memblock {
	struct memblock *next;
	size_t size;
	void *buff;
	int64_t filebytes;
	};

	/*
	* The total memory store is just a list of memblocks plus
	* some accounting overhead.
	*/
	struct memdata {
	int64_t filebytes;
	void *buff;
	struct memblock *first;
	struct memblock *last;
	};

	/* The following size definitions simplify things below. */
	#define KB ((int64_t)1024)
	#define MB ((int64_t)1024 * KB)
	#define GB ((int64_t)1024 * MB)
	#define TB ((int64_t)1024 * GB)

	static int64_t memory_read_skip(struct archive , void , int64_t request);
	static ssize_t memory_read(struct archive , void , const void **buff);
	static ssize_t memory_write(struct archive , void , const void *, size_t);


	static ssize_t
	memory_write(struct archive a, void _private, const void *buff, size_t size)
	{
	struct memdata *private = _private;
	struct memblock *block;

	(void)a;

	/*
	* Since libarchive tries to behave in a zero-copy manner, if
	* you give a pointer to filedata to the library, a pointer
	* into that data will (usually) pop out here. This way, we
	* can tell the difference between filedata and library header
	* and metadata.
	*/
	if ((const char )filedata <= (const char )buff
	&& (const char )buff < (const char )filedata + filedatasize) {
	/* We don't need to store a block of file data. */
	private->last->filebytes += (int64_t)size;
	} else {
	/* Yes, we're assuming the very first write is metadata. */
	/* It's header or metadata, copy and save it. */
	block = (struct memblock )malloc(sizeof(block));
	memset(block, 0, sizeof(*block));
	block->size = size;
	block->buff = malloc(size);
	memcpy(block->buff, buff, size);
	if (private->last == NULL) {
	private->first = private->last = block;
	} else {
	private->last->next = block;
	private->last = block;
	}
	block->next = NULL;
	}
	return ((long)size);
	}

	static ssize_t
	memory_read(struct archive a, void _private, const void **buff)
	{
	struct memdata *private = _private;
	struct memblock *block;
	ssize_t size;

	(void)a;

	free(private->buff);
	private->buff = NULL;
	if (private->first == NULL) {
	private->last = NULL;
	return (ARCHIVE_EOF);
	}
	if (private->filebytes > 0) {
	/*
	* We're returning file bytes, simulate it by
	* passing blocks from the template data.
	*/
	if (private->filebytes > (int64_t)filedatasize)
	size = (ssize_t)filedatasize;
	else
	size = (ssize_t)private->filebytes;
	private->filebytes -= size;
	*buff = filedata;
	} else {
	/*
	* We need to get some real data to return.
	*/
	block = private->first;
	private->first = block->next;
	size = (ssize_t)block->size;
	if (block->buff != NULL) {
	private->buff = block->buff;
	*buff = block->buff;
	} else {
	private->buff = NULL;
	*buff = filedata;
	}
	private->filebytes = block->filebytes;
	free(block);
	}
	return (size);
	}


	static int64_t
	memory_read_skip(struct archive a, void _private, int64_t skip)
	{
	struct memdata *private = _private;

	(void)a;

	if (private->first == NULL) {
	private->last = NULL;
	return (0);
	}
	if (private->filebytes > 0) {
	if (private->filebytes < skip)
	skip = (off_t)private->filebytes;
	private->filebytes -= skip;
	} else {
	skip = 0;
	}
	return (skip);
	}

	DEFINE_TEST(test_tar_large)
	{
	/* The sizes of the entries we're going to generate. */
	static int64_t tests[] = {
	/* Test for 32-bit signed overflow. */
	2 * GB - 1, 2 * GB, 2 * GB + 1,
	/* Test for 32-bit unsigned overflow. */
	4 * GB - 1, 4 * GB, 4 * GB + 1,
	/* 8GB is the "official" max for ustar. */
	8 * GB - 1, 8 * GB, 8 * GB + 1,
	/* Bend ustar a tad and you can get 64GB (12 octal digits). */
	64 * GB - 1, 64 * GB,
	/* And larger entries that require non-ustar extensions. */
	256 * GB, 1 * TB, 0 };
	int i;
	char namebuff[64];
	struct memdata memdata;
	struct archive_entry *ae;
	struct archive *a;
	int64_t filesize;
	size_t writesize;

	filedatasize = (size_t)(1 * MB);
	filedata = malloc(filedatasize);
	memset(filedata, 0xAA, filedatasize);
	memset(&memdata, 0, sizeof(memdata));

	/*
	* Open an archive for writing.
	*/
	a = archive_write_new();
	archive_write_set_format_pax_restricted(a);
	archive_write_set_bytes_per_block(a, 0); /* No buffering. */
	archive_write_open(a, &memdata, NULL, memory_write, NULL);

	/*
	* Write a series of large files to it.
	*/
	for (i = 0; tests[i] != 0; i++) {
	assert((ae = archive_entry_new()) != NULL);
	sprintf(namebuff, "file_%d", i);
	archive_entry_copy_pathname(ae, namebuff);
	archive_entry_set_mode(ae, S_IFREG \| 0755);
	filesize = tests[i];

	archive_entry_set_size(ae, filesize);

	assertA(0 == archive_write_header(a, ae));
	archive_entry_free(ae);

	/*
	* Write the actual data to the archive.
	*/
	while (filesize > 0) {
	writesize = filedatasize;
	if ((int64_t)writesize > filesize)
	writesize = (size_t)filesize;
	assertA((int)writesize
	== archive_write_data(a, filedata, writesize));
	filesize -= writesize;
	}
	}

	assert((ae = archive_entry_new()) != NULL);
	archive_entry_copy_pathname(ae, "lastfile");
	archive_entry_set_mode(ae, S_IFREG \| 0755);
	assertA(0 == archive_write_header(a, ae));
	archive_entry_free(ae);


	/* Close out the archive. */
	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
	assertEqualInt(ARCHIVE_OK, archive_write_free(a));

	/*
	* Open the same archive for reading.
	*/
	a = archive_read_new();
	archive_read_support_format_tar(a);
	archive_read_open2(a, &memdata, NULL,
	memory_read, memory_read_skip, NULL);

	/*
	* Read entries back.
	*/
	for (i = 0; tests[i] > 0; i++) {
	assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
	sprintf(namebuff, "file_%d", i);
	assertEqualString(namebuff, archive_entry_pathname(ae));
	assert(tests[i] == archive_entry_size(ae));
	}
	assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
	assertEqualString("lastfile", archive_entry_pathname(ae));

	assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));

	/* Close out the archive. */
	assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
	assertEqualInt(ARCHIVE_OK, archive_read_free(a));

	free(memdata.buff);
	free(filedata);
	}