Skip to content

Commit

Permalink
now upload APIs could switch upHosts
Browse files Browse the repository at this point in the history
  • Loading branch information
@bachue
bachue committed Apr 17, 2024
1 parent 2b6d3ed commit 982b978
Show file tree
Hide file tree
Showing 14 changed files with 464 additions and 218 deletions.
12 changes: 12 additions & 0 deletions qiniu/base.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,18 @@ void Qiniu_FreeV2(void **addr)
*addr = NULL;
}
}
void Qiniu_Multi_Free(int n, ...)
{
void *p = NULL;
va_list v1;
va_start(v1, n);
for (int i = 0; i < n; i++)
{
p = va_arg(v1, void *);
free(p);
}
va_end(v1);
}

/*============================================================================*/
/* type Qiniu_Count */
Expand Down
1 change: 1 addition & 0 deletions qiniu/base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ typedef unsigned long long Qiniu_Uint64;

QINIU_DLLAPI extern void Qiniu_Free(void *addr);
QINIU_DLLAPI extern void Qiniu_FreeV2(void **addr);
QINIU_DLLAPI extern void Qiniu_Multi_Free(int n, ...);
/*============================================================================*/
/* type Qiniu_Count */

Expand Down
42 changes: 42 additions & 0 deletions qiniu/code.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#include <curl/curl.h>
#include "private/code.h"

Qiniu_Retry_Decision _Qiniu_Should_Retry(int code)
{
if (code / 100 == 4 && code != 406 && code != 429)
{
return QINIU_DONT_RETRY;
}
switch (code)
{
case CURLE_COULDNT_RESOLVE_HOST:
case CURLE_COULDNT_CONNECT:
case CURLE_WEIRD_SERVER_REPLY:
case CURLE_PARTIAL_FILE:
case CURLE_UPLOAD_FAILED:
case CURLE_OPERATION_TIMEDOUT:
case CURLE_SSL_CONNECT_ERROR:
case CURLE_SEND_ERROR:
case CURLE_RECV_ERROR:
case CURLE_SSL_CERTPROBLEM:
case CURLE_SSL_CIPHER:
case CURLE_PEER_FAILED_VERIFICATION:
case CURLE_HTTP2_STREAM:
case CURLE_HTTP3:
case CURLE_QUIC_CONNECT_ERROR:
case 406:
case 429:
case 500:
case 502:
case 503:
case 504:
case 509:
case 571:
case 573:
case 599:
return QINIU_TRY_NEXT_DOMAIN;

default:
return QINIU_DONT_RETRY;
}
}
8 changes: 7 additions & 1 deletion qiniu/http.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize)
Qiniu_Zero_Ptr(self);
self->curl = curl_easy_init();
self->auth = auth;
self->retriesMax = SIZE_MAX;

Qiniu_Buffer_Init(&self->b, bufSize);
Qiniu_Buffer_Init(&self->respHeader, bufSize);
Expand Down Expand Up @@ -396,6 +397,11 @@ void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long
self->lowSpeedTime = lowSpeedTime;
} // Qiniu_Client_SetLowSpeedLimit

void Qiniu_Client_SetMaximumRetries(Qiniu_Client *self, size_t retriesMax)
{
self->retriesMax = retriesMax;
} // Qiniu_Client_SetMaximumRetries

void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs)
{
self->timeoutMs = timeoutMs;
Expand Down Expand Up @@ -475,7 +481,7 @@ static Qiniu_Error Qiniu_Client_callWithBody(
Qiniu_Error err;
const char *ctxType;
char ctxLength[64], userAgent[64];
Qiniu_Header *headers = NULL;
Qiniu_Header *headers;
CURL *curl = (CURL *)self->curl;
err = Qiniu_Client_config(self);
if (err.code != Qiniu_OK.code)
Expand Down
4 changes: 4 additions & 0 deletions qiniu/http.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,13 +139,17 @@ extern "C"

// Millisecond timeout for the connection phase.
long connectTimeoutMs;

// Max retries count.
size_t retriesMax;
};
typedef struct _Qiniu_Client Qiniu_Client;

QINIU_DLLAPI extern void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize);
QINIU_DLLAPI extern void Qiniu_Client_Cleanup(Qiniu_Client *self);
QINIU_DLLAPI extern void Qiniu_Client_BindNic(Qiniu_Client *self, const char *nic);
QINIU_DLLAPI extern void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long lowSpeedTime);
QINIU_DLLAPI extern void Qiniu_Client_SetMaximumRetries(Qiniu_Client *self, size_t retries);
QINIU_DLLAPI extern void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs);
QINIU_DLLAPI extern void Qiniu_Client_SetConnectTimeout(Qiniu_Client *self, long connectTimeoutMs);
QINIU_DLLAPI extern void Qiniu_Client_EnableAutoQuery(Qiniu_Client *self, Qiniu_Bool useHttps);
Expand Down
139 changes: 87 additions & 52 deletions qiniu/io.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "reader.h"
#include "recorder_utils.h"
#include "private/region.h"
#include "private/code.h"
#include <curl/curl.h>

/*============================================================================*/
Expand Down Expand Up @@ -58,20 +59,27 @@ CURL *Qiniu_Client_reset(Qiniu_Client *self);

Qiniu_Error Qiniu_callex(CURL *curl, Qiniu_Buffer *resp, Qiniu_Json **ret, Qiniu_Bool simpleError, Qiniu_Buffer *resph);

static Qiniu_Error Get_Qiniu_UpHost(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char **upHost)
static Qiniu_Error Get_Qiniu_UpHosts(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char *const **upHosts, size_t *upHostsCount)
{
if (extra && extra->ipCount != 0)
{
Qiniu_Count oldIndex = Qiniu_Count_Inc(&extra->ipIndex);
*upHost = extra->upIps[abs(oldIndex % extra->ipCount)];
*upHosts = &extra->upIps[abs(oldIndex % extra->ipCount)];
*upHostsCount = 1;
}
else if (extra && extra->upHost != NULL)
{
*upHost = extra->upHost;
*upHosts = &extra->upHost;
*upHostsCount = 1;
}
else if (extra && extra->upHosts != NULL && extra->upHostsCount != 0)
{
*upHosts = extra->upHosts;
*upHostsCount = extra->upHostsCount;
}
else
{
Qiniu_Error err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, upHost);
Qiniu_Error err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, upHosts, upHostsCount);
if (err.code != Qiniu_OK.code)
{
return err;
Expand Down Expand Up @@ -150,48 +158,61 @@ static Qiniu_Error Qiniu_Io_call(
{
int retCode = 0;
Qiniu_Error err;
struct curl_slist *headers = NULL;
const char *upHost = NULL;
struct curl_slist *headers;
const char *const *upHosts;
const char *defaultUpHosts[] = {QINIU_UP_HOST};
size_t upHostsCount;

err = Qiniu_Client_config(self);
headers = curl_slist_append(NULL, "Expect:");
err = Get_Qiniu_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount);
if (err.code != Qiniu_OK.code)
{
return err;
}
err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost);
if (err.code != Qiniu_OK.code)
if (upHostsCount == 0)
{
return err;
upHosts = defaultUpHosts;
upHostsCount = 1;
}

headers = curl_slist_append(NULL, "Expect:");

CURL *curl = Qiniu_Client_reset(self);
curl_easy_setopt(curl, CURLOPT_URL, upHost);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

//// For aborting uploading file.
if (extra->upAbortCallback)
for (size_t retries = 0; retries < upHostsCount && retries <= self->retriesMax; retries++)
{
curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback);
} // if
CURL *curl = Qiniu_Client_reset(self);
err = Qiniu_Client_config(self);
if (err.code != Qiniu_OK.code)
{
return err;
}
curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries]);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
if (err.code == Qiniu_OK.code && ret != NULL)
{
if (extra->callbackRetParser != NULL)
//// For aborting uploading file.
if (extra->upAbortCallback)
{
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback);
} // if

err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
if (err.code == Qiniu_OK.code)
{
if (extra->callbackRetParser != NULL)
{
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
}
else if (ret != NULL)
{
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL);
}
break;
}
else
else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
{
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL);
break;
}
}

curl_formfree(formpost);
curl_slist_free_all(headers);
return err;
Expand Down Expand Up @@ -327,42 +348,56 @@ static Qiniu_Error Qiniu_Io_call_with_callback(
{
int retCode = 0;
Qiniu_Error err;
struct curl_slist *headers = NULL;
const char *upHost;
struct curl_slist *headers;
const char *const *upHosts;
const char *defaultUpHosts[] = {QINIU_UP_HOST};
size_t upHostsCount;

CURL *curl = Qiniu_Client_reset(self);
err = Qiniu_Client_config(self);
headers = curl_slist_append(NULL, "Expect:");
err = Get_Qiniu_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount);
if (err.code != Qiniu_OK.code)
{
return err;
}
err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost);
if (err.code != Qiniu_OK.code)
if (upHostsCount == 0)
{
return err;
upHosts = defaultUpHosts;
upHostsCount = 1;
}

headers = curl_slist_append(NULL, "Expect:");
for (size_t retries = 0; retries < upHostsCount && retries <= self->retriesMax; retries++)
{
CURL *curl = Qiniu_Client_reset(self);
err = Qiniu_Client_config(self);
if (err.code != Qiniu_OK.code)
{
return err;
}

curl_easy_setopt(curl, CURLOPT_URL, upHost);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);
curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries]);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);

err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
if (err.code == Qiniu_OK.code && ret != NULL)
{
if (extra->callbackRetParser != NULL)
err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
if (err.code == Qiniu_OK.code)
{
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
if (extra->callbackRetParser != NULL)
{
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
}
else if (ret != NULL)
{
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
}
break;
}
else
else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
{
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
break;
}
}

curl_formfree(formpost);
curl_slist_free_all(headers);
return err;
Expand Down
7 changes: 6 additions & 1 deletion qiniu/io.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
Name : io.h
Author : Qiniu.com
Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd.
Description :
Description :
============================================================================
*/

Expand Down Expand Up @@ -51,9 +51,14 @@ extern "C"
Qiniu_Rd_FnAbort upAbortCallback;

const char *upHost;

const char **upIps;
Qiniu_Count ipCount;
Qiniu_Count ipIndex;

// Specify multiple upHosts
const char *const *upHosts;
size_t upHostsCount;
} Qiniu_Io_PutExtra;

/*============================================================================*/
Expand Down
Loading

0 comments on commit 982b978

Please sign in to comment.