mksquashfs: fix recursive restore failure check

Fix recursive failure in restoring check by moving it to
prep_exit() and from restorefs(), this is because due to the
changes where all restoring is now done on the restore thread,
restorefs() is no longer recursively called, only prep_exit()
is now recursively called.

Whilst doing this, take advantage of the situation where restoring
is now only done on the restore thread to fix a theorectical
race condition with the previous code.  The previous code identified
recursive failure by incrementing the restoring flag every time it
was called, if the restore flag increased beyond one then we had
recursion, unfortunately this did not handle the case where two or more
threads simultaneously entered the restoring code but before restoring
took place, each thread would increment the flag making it look like
recursive restoring had taken place.  The new code fixes this by
checking if the restore thread has entered prep_exit(), if it has
then we have recursive failure.  This is a much more specific check
which allows multiple threads to fail and enter the restore code
simultaneously.

Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
index 386c8ed..8c252ac 100644
--- a/squashfs-tools/mksquashfs.c
+++ b/squashfs-tools/mksquashfs.c
@@ -229,9 +229,6 @@
 	char			checksum_flag;
 };
 
-/* flag if we're restoring existing filesystem */
-int restoring = 0;
-
 /* restore orignal filesystem state if appending to existing filesystem is
  * cancelled */
 char *sdata_cache, *sdirectory_data_cache, *sdirectory_compressed;
@@ -244,7 +241,6 @@
 	sinode_count = 0, sfile_count, ssym_count, sdev_count,
 	sdir_count, sfifo_count, ssock_count, sdup_files;
 int sfragments;
-int restore = 0;
 int threads;
 
 /* flag whether destination file is a block device */
@@ -315,6 +311,7 @@
 struct queue *to_reader, *from_reader, *to_writer, *from_writer, *from_deflate,
 	*to_frag;
 pthread_t *thread, *deflator_thread, *frag_deflator_thread;
+pthread_t *restore_thread = NULL;
 pthread_mutex_t	fragment_mutex;
 pthread_cond_t fragment_waiting;
 pthread_mutex_t	pos_mutex;
@@ -381,16 +378,27 @@
 struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
 extern void init_info();
 extern void update_info(struct dir_ent *);
-extern void init_restore_thread(pthread_t);
+extern pthread_t *init_restore_thread(pthread_t);
 void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
 
 
 void prep_exit()
 {
-	if(restore) {
-		/* signal the restore thread to restore */
-		kill(getpid(), SIGUSR1);
-		pthread_exit(NULL);
+	if(restore_thread) {
+		if(pthread_self() == *restore_thread) {
+			/*
+			 * Recursive failure when trying to restore filesystem!
+			 * Nothing to do except to exit, otherwise we'll just
+			 * appear to hang.  The user should be able to restore
+			 * from the recovery file (which is why it was added, in
+			 * case of catastrophic failure in Mksquashfs)
+			 */
+			exit(1);
+		} else {
+			/* signal the restore thread to restore */
+			kill(getpid(), SIGUSR1);
+			pthread_exit(NULL);
+		}
 	}
 	if(delete && destination_file && !block_device)
 		unlink(destination_file);
@@ -713,16 +721,6 @@
 {
 	int i;
 
-	if(restoring++)
-		/*
-		 * Recursive failure when trying to restore filesystem!
-		 * Nothing to do except to exit, otherwise we'll just appear
-		 * to hang.  The user should be able to restore from the
-		 * recovery file (which is why it was added, in case of
-		 * catastrophic failure in Mksquashfs)
-		 */
-		exit(1);
-
 	ERROR("Exiting - restoring original filesystem!\n\n");
 
 	for(i = 0; i < 2 + processors * 2; i++)
@@ -5085,7 +5083,7 @@
 
 
 #define VERSION() \
-	printf("mksquashfs version 4.2-git (2013/04/05)\n");\
+	printf("mksquashfs version 4.2-git (2013/04/07)\n");\
 	printf("copyright (C) 2013 Phillip Lougher "\
 		"<phillip@squashfs.org.uk>\n\n"); \
 	printf("This program is free software; you can redistribute it and/or"\
@@ -5765,8 +5763,7 @@
 		sid_count = id_count;
 		write_recovery_data(&sBlk);
 		save_xattrs();
-		restore = TRUE;
-		init_restore_thread(pthread_self());
+		restore_thread = init_restore_thread(pthread_self());
 		sigemptyset(&sigmask);
 		sigaddset(&sigmask, SIGINT);
 		sigaddset(&sigmask, SIGTERM);
diff --git a/squashfs-tools/restore.c b/squashfs-tools/restore.c
index 71827c1..5807167 100644
--- a/squashfs-tools/restore.c
+++ b/squashfs-tools/restore.c
@@ -82,8 +82,9 @@
 }
 
 
-void init_restore_thread(pthread_t thread)
+pthread_t *init_restore_thread(pthread_t thread)
 {
 	main_thread = thread;
 	pthread_create(&restore_thread, NULL, restore_thrd, NULL);
+	return &restore_thread;
 }