From: Heng Li <lh3@me.com>
Date: Tue, 19 Nov 2013 03:07:40 +0000 (-0500)
Subject: need to grow buffer
X-Git-Url: http://www.kaiwu.me/postgresql/commit/?a=commitdiff_plain;h=1666c827932c415c627e47555f88ada66755159f;p=klib.git

need to grow buffer

One multi_perform() frequently invokes multiple write_cb(). We have to use a
dynamic buffer. Hmm.. I really do not like the curl APIs, although I understand
why they are designed this way.
---

diff --git a/kurl.c b/kurl.c
index 4fcf103..7f5dd9c 100644
--- a/kurl.c
+++ b/kurl.c
@@ -14,6 +14,10 @@
 
 #define kurl_isfile(u) ((u)->fd >= 0)
 
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
 struct kurl_t {
 	CURLM *multi; // cURL multi handler
 	CURL *curl;   // cURL easy handle
@@ -56,7 +60,15 @@ static size_t write_cb(char *ptr, size_t size, size_t nmemb, void *data) // call
 {
 	kurl_t *ku = (kurl_t*)data;
 	ssize_t nbytes = size * nmemb;
-	assert(nbytes + ku->l_buf < ku->m_buf);
+	if (nbytes + ku->l_buf > ku->m_buf) {
+		ku->m_buf = nbytes + ku->l_buf;
+		kroundup32(ku->m_buf);
+		ku->buf = (uint8_t*)realloc(ku->buf, ku->m_buf);
+		if (ku->buf == 0) {
+			ku->err = KURL_NO_MEM;
+			return 0;
+		}
+	}
 	memcpy(ku->buf + ku->l_buf, ptr, nbytes);
 	ku->l_buf += nbytes;
 	return nbytes;
@@ -103,8 +115,8 @@ static int fill_buffer(kurl_t *ku) // fill the buffer
 				nanosleep(&req, &rem);
 			}
 			rc = curl_multi_perform(ku->multi, &n_running); // FIXME: check return code
-		} while (n_running && ku->l_buf < ku->m_buf - CURL_MAX_WRITE_SIZE);
-		if (ku->l_buf < ku->m_buf - CURL_MAX_WRITE_SIZE) ku->done_reading = 1;
+		} while (n_running && ku->l_buf < CURL_MAX_WRITE_SIZE);
+		if (ku->l_buf < CURL_MAX_WRITE_SIZE) ku->done_reading = 1;
 	}
 	return ku->l_buf;
 }
@@ -173,10 +185,6 @@ kurl_t *kurl_dopen(int fd)
 	return ku;
 }
 
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
 int kurl_buflen(kurl_t *ku, int len)
 {
 	if (len <= 0 || len < ku->l_buf) return ku->m_buf;
diff --git a/kurl.h b/kurl.h
index f31fd9d..a4a6487 100644
--- a/kurl.h
+++ b/kurl.h
@@ -6,6 +6,7 @@
 #define KURL_NULL       1
 #define KURL_INV_WHENCE 2
 #define KURL_SEEK_OUT   3
+#define KURL_NO_MEM     4
 
 struct kurl_t;
 typedef struct kurl_t kurl_t;
@@ -43,6 +44,7 @@ typedef kurl_t knetFile;
 #define knet_open(fn, mode) kurl_open(fn, 0)
 #define knet_dopen(fd, mode) kurl_dopen(fd)
 #define knet_close(fp) kurl_close(fp)
+#define knet_read(fp, buf, len) kurl_read(fp, buf, len)
 #define knet_seek(fp, off, whence) kurl_seek(fp, off, whence)
 #define knet_tell(fp) kurl_tell(fp)
 #define knet_fileno(fp) kurl_fileno(fp)