need to grow buffer

One multi_perform() frequently invokes multiple write_cb(). We have to use a
dynamic buffer. Hmm.. I really do not like the curl APIs, although I understand
why they are designed this way.

diff --git a/kurl.c b/kurl.c
index 4fcf1034e265a1f0d2be50f95e26b74289428237..7f5dd9cb6352a0b677fb0c7ceed41145535b53bc 100644 (file)
--- a/kurl.c
+++ b/kurl.c
@@ -14,6 +14,10 @@
 
 #define kurl_isfile(u) ((u)->fd >= 0)
 
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
 struct kurl_t {
        CURLM *multi; // cURL multi handler
        CURL *curl;   // cURL easy handle
@@ -56,7 +60,15 @@ static size_t write_cb(char *ptr, size_t size, size_t nmemb, void *data) // call
 {
        kurl_t *ku = (kurl_t*)data;
        ssize_t nbytes = size * nmemb;
-       assert(nbytes + ku->l_buf < ku->m_buf);
+       if (nbytes + ku->l_buf > ku->m_buf) {
+               ku->m_buf = nbytes + ku->l_buf;
+               kroundup32(ku->m_buf);
+               ku->buf = (uint8_t*)realloc(ku->buf, ku->m_buf);
+               if (ku->buf == 0) {
+                       ku->err = KURL_NO_MEM;
+                       return 0;
+               }
+       }
        memcpy(ku->buf + ku->l_buf, ptr, nbytes);
        ku->l_buf += nbytes;
        return nbytes;
@@ -103,8 +115,8 @@ static int fill_buffer(kurl_t *ku) // fill the buffer
                                nanosleep(&req, &rem);
                        }
                        rc = curl_multi_perform(ku->multi, &n_running); // FIXME: check return code
-               } while (n_running && ku->l_buf < ku->m_buf - CURL_MAX_WRITE_SIZE);
-               if (ku->l_buf < ku->m_buf - CURL_MAX_WRITE_SIZE) ku->done_reading = 1;
+               } while (n_running && ku->l_buf < CURL_MAX_WRITE_SIZE);
+               if (ku->l_buf < CURL_MAX_WRITE_SIZE) ku->done_reading = 1;
        }
        return ku->l_buf;
 }
@@ -173,10 +185,6 @@ kurl_t *kurl_dopen(int fd)
        return ku;
 }
 
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
 int kurl_buflen(kurl_t *ku, int len)
 {
        if (len <= 0 || len < ku->l_buf) return ku->m_buf;

diff --git a/kurl.h b/kurl.h
index f31fd9d94d744936101f8028eb25b7d082539748..a4a6487a382fa5a15f8077d93c972bcf4b5ddd3c 100644 (file)
--- a/kurl.h
+++ b/kurl.h
@@ -6,6 +6,7 @@
 #define KURL_NULL       1
 #define KURL_INV_WHENCE 2
 #define KURL_SEEK_OUT   3
+#define KURL_NO_MEM     4
 
 struct kurl_t;
 typedef struct kurl_t kurl_t;
@@ -43,6 +44,7 @@ typedef kurl_t knetFile;
 #define knet_open(fn, mode) kurl_open(fn, 0)
 #define knet_dopen(fd, mode) kurl_dopen(fd)
 #define knet_close(fp) kurl_close(fp)
+#define knet_read(fp, buf, len) kurl_read(fp, buf, len)
 #define knet_seek(fp, off, whence) kurl_seek(fp, off, whence)
 #define knet_tell(fp) kurl_tell(fp)
 #define knet_fileno(fp) kurl_fileno(fp)