Allow formposting of files larger than what fits in memory by not reading the
file until it is actually being uploaded.
Make sure we build and still work with HTTP disabled - the SSL code might use
the boundary string for some random seeding.
diff --git a/lib/formdata.c b/lib/formdata.c
index 8fb6d9c..ea5c4ac 100644
--- a/lib/formdata.c
+++ b/lib/formdata.c
@@ -102,6 +102,9 @@
#include "setup.h"
#include <curl/curl.h>
+/* Length of the random boundary string. */
+#define BOUNDARY_LENGTH 40
+
#ifndef CURL_DISABLE_HTTP
#include <stdio.h>
@@ -109,6 +112,7 @@
#include <string.h>
#include <stdarg.h>
#include <time.h>
+#include <sys/stat.h>
#include "formdata.h"
#include "strequal.h"
#include "memory.h"
@@ -119,9 +123,6 @@
/* The last #include file should be: */
#include "memdebug.h"
-/* Length of the random boundary string. */
-#define BOUNDARY_LENGTH 40
-
/* What kind of Content-Type to use on un-specified files with unrecognized
extensions. */
#define HTTPPOST_CONTENTTYPE_DEFAULT "application/octet-stream"
@@ -785,9 +786,10 @@
* size is incremented by the chunk length, unless it is NULL
*/
static CURLcode AddFormData(struct FormData **formp,
+ enum formtype type,
const void *line,
size_t length,
- size_t *size)
+ curl_off_t *size)
{
struct FormData *newform = (struct FormData *)
malloc(sizeof(struct FormData));
@@ -807,6 +809,7 @@
memcpy(newform->line, line, length);
newform->length = length;
newform->line[length]=0; /* zero terminate for easier debugging */
+ newform->type = type;
if(*formp) {
(*formp)->next = newform;
@@ -815,8 +818,20 @@
else
*formp = newform;
- if (size)
- *size += length;
+ if (size) {
+ if(type == FORM_DATA)
+ *size += length;
+ else {
+ /* Since this is a file to be uploaded here, add the size of the actual
+ file */
+ if(!strequal("-", newform->line)) {
+ struct stat file;
+ if(!stat(newform->line, &file)) {
+ *size += file.st_size;
+ }
+ }
+ }
+ }
return CURLE_OK;
}
@@ -825,7 +840,7 @@
*/
static CURLcode AddFormDataf(struct FormData **formp,
- size_t *size,
+ curl_off_t *size,
const char *fmt, ...)
{
char s[4096];
@@ -834,39 +849,7 @@
vsprintf(s, fmt, ap);
va_end(ap);
- return AddFormData(formp, s, 0, size);
-}
-
-/*
- * Curl_FormBoundary() creates a suitable boundary string and returns an
- * allocated one.
- */
-char *Curl_FormBoundary(void)
-{
- char *retstring;
- static int randomizer=0; /* this is just so that two boundaries within
- the same form won't be identical */
- size_t i;
-
- static char table16[]="abcdef0123456789";
-
- retstring = (char *)malloc(BOUNDARY_LENGTH+1);
-
- if(!retstring)
- return NULL; /* failed */
-
- srand(time(NULL)+randomizer++); /* seed */
-
- strcpy(retstring, "----------------------------");
-
- for(i=strlen(retstring); i<BOUNDARY_LENGTH; i++)
- retstring[i] = table16[rand()%16];
-
- /* 28 dashes and 12 hexadecimal digits makes 12^16 (184884258895036416)
- combinations */
- retstring[BOUNDARY_LENGTH]=0; /* zero terminate */
-
- return retstring;
+ return AddFormData(formp, FORM_DATA, s, 0, size);
}
/*
@@ -936,7 +919,7 @@
struct curl_httppost *file;
CURLcode result = CURLE_OK;
- size_t size =0;
+ curl_off_t size=0; /* support potentially ENORMOUS formposts */
char *boundary;
char *fileboundary=NULL;
struct curl_slist* curList;
@@ -977,16 +960,17 @@
if (result)
break;
- result = AddFormData(&form,
- "Content-Disposition: form-data; name=\"", 0, &size);
+ result = AddFormDataf(&form, &size,
+ "Content-Disposition: form-data; name=\"");
if (result)
break;
- result = AddFormData(&form, post->name, post->namelength, &size);
+ result = AddFormData(&form, FORM_DATA, post->name, post->namelength,
+ &size);
if (result)
break;
- result = AddFormData(&form, "\"", 0, &size);
+ result = AddFormDataf(&form, &size, "\"");
if (result)
break;
@@ -1071,7 +1055,7 @@
}
#endif
- result = AddFormData(&form, "\r\n\r\n", 0, &size);
+ result = AddFormDataf(&form, &size, "\r\n\r\n");
if (result)
break;
@@ -1079,11 +1063,10 @@
(post->flags & HTTPPOST_READFILE)) {
/* we should include the contents from the specified file */
FILE *fileread;
- char buffer[1024];
- size_t nread;
fileread = strequal("-", file->contents)?
stdin:fopen(file->contents, "rb"); /* binary read for win32 */
+
/*
* VMS: This only allows for stream files on VMS. Stream files are
* OK, as are FIXED & VAR files WITHOUT implied CC For implied CC,
@@ -1091,13 +1074,27 @@
*/
if(fileread) {
- while((nread = fread(buffer, 1, 1024, fileread))) {
- result = AddFormData(&form, buffer, nread, &size);
- if (result)
- break;
- }
- if(fileread != stdin)
+ if(fileread != stdin) {
+ /* close the file again */
fclose(fileread);
+ /* add the file name only - for later reading from this */
+ result = AddFormData(&form, FORM_FILE, file->contents, 0, &size);
+ }
+ else {
+ /* When uploading from stdin, we can't know the size of the file,
+ * thus must read the full file as before. We *could* use chunked
+ * transfer-encoding, but that only works for HTTP 1.1 and we
+ * can't be sure we work with such a server.
+ */
+ size_t nread;
+ char buffer[512];
+ while((nread = fread(buffer, 1, sizeof(buffer), fileread))) {
+ result = AddFormData(&form, FORM_DATA, buffer, nread, &size);
+ if (result)
+ break;
+ }
+ }
+
if (result) {
Curl_formclean(firstform);
free(boundary);
@@ -1115,16 +1112,16 @@
}
else if (post->flags & HTTPPOST_BUFFER) {
/* include contents of buffer */
- result = AddFormData(&form, post->buffer, post->bufferlength,
- &size);
+ result = AddFormData(&form, FORM_DATA, post->buffer,
+ post->bufferlength, &size);
if (result)
break;
}
else {
/* include the contents we got */
- result = AddFormData(&form, post->contents, post->contentslength,
- &size);
+ result = AddFormData(&form, FORM_DATA, post->contents,
+ post->contentslength, &size);
if (result)
break;
}
@@ -1183,10 +1180,32 @@
form->data = formdata;
form->sent = 0;
+ form->fp = NULL;
return 0;
}
+static size_t readfromfile(struct Form *form, char *buffer, size_t size)
+{
+ size_t nread;
+ if(!form->fp) {
+ /* this file hasn't yet been opened */
+ form->fp = fopen(form->data->line, "rb"); /* b is for binary */
+ if(!form->fp)
+ return -1; /* failure */
+ }
+ nread = fread(buffer, 1, size, form->fp);
+
+ if(nread != size) {
+ /* this is the last chunk form the file, move on */
+ fclose(form->fp);
+ form->fp = NULL;
+ form->data = form->data->next;
+ }
+
+ return nread;
+}
+
/*
* Curl_FormReader() is the fread() emulation function that will be used to
* deliver the formdata to the transfer loop and then sent away to the peer.
@@ -1207,6 +1226,9 @@
if(!form->data)
return 0; /* nothing, error, empty */
+ if(form->data->type == FORM_FILE)
+ return readfromfile(form, buffer, wantedsize);
+
do {
if( (form->data->length - form->sent ) > wantedsize - gotsize) {
@@ -1228,7 +1250,7 @@
form->data = form->data->next; /* advance */
- } while(form->data);
+ } while(form->data && (form->data->type == FORM_DATA));
/* If we got an empty line and we have more data, we proceed to the next
line immediately to avoid returning zero before we've reached the end.
This is the bug reported November 22 1999 on curl 6.3. (Daniel) */
@@ -1475,3 +1497,36 @@
}
#endif /* CURL_DISABLE_HTTP */
+
+/*
+ * Curl_FormBoundary() creates a suitable boundary string and returns an
+ * allocated one. This is also used by SSL-code so it must be present even
+ * if HTTP is disabled!
+ */
+char *Curl_FormBoundary(void)
+{
+ char *retstring;
+ static int randomizer=0; /* this is just so that two boundaries within
+ the same form won't be identical */
+ size_t i;
+
+ static char table16[]="abcdef0123456789";
+
+ retstring = (char *)malloc(BOUNDARY_LENGTH+1);
+
+ if(!retstring)
+ return NULL; /* failed */
+
+ srand(time(NULL)+randomizer++); /* seed */
+
+ strcpy(retstring, "----------------------------");
+
+ for(i=strlen(retstring); i<BOUNDARY_LENGTH; i++)
+ retstring[i] = table16[rand()%16];
+
+ /* 28 dashes and 12 hexadecimal digits makes 12^16 (184884258895036416)
+ combinations */
+ retstring[BOUNDARY_LENGTH]=0; /* zero terminate */
+
+ return retstring;
+}
diff --git a/lib/formdata.h b/lib/formdata.h
index eb5284c..c6a78cd 100644
--- a/lib/formdata.h
+++ b/lib/formdata.h
@@ -2,10 +2,10 @@
#define __FORMDATA_H
/***************************************************************************
- * _ _ ____ _
- * Project ___| | | | _ \| |
- * / __| | | | |_) | |
- * | (__| |_| | _ <| |___
+ * _ _ ____ _
+ * Project ___| | | | _ \| |
+ * / __| | | | |_) | |
+ * | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2004, Daniel Stenberg, <daniel@haxx.se>, et al.
@@ -13,7 +13,7 @@
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
- *
+ *
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
@@ -23,17 +23,25 @@
*
* $Id$
***************************************************************************/
+
+enum formtype {
+ FORM_DATA, /* regular data */
+ FORM_FILE /* 'line' points to a file name we should read from */
+};
+
/* plain and simple linked list with lines to send */
struct FormData {
struct FormData *next;
+ enum formtype type;
char *line;
size_t length;
};
struct Form {
struct FormData *data; /* current form line to send */
- unsigned int sent; /* number of bytes of the current line that has already
- been sent in a previous invoke */
+ size_t sent; /* number of bytes of the current line that has
+ already been sent in a previous invoke */
+ FILE *fp; /* file to read from */
};
/* used by FormAdd for temporary storage */
