diff --git a/qiniu/base.c b/qiniu/base.c index a0f36ac..1bb6555 100644 --- a/qiniu/base.c +++ b/qiniu/base.c @@ -31,6 +31,18 @@ void Qiniu_FreeV2(void **addr) *addr = NULL; } } +void Qiniu_Multi_Free(int n, ...) +{ + void *p = NULL; + va_list v1; + va_start(v1, n); + for (int i = 0; i < n; i++) + { + p = va_arg(v1, void *); + free(p); + } + va_end(v1); +} /*============================================================================*/ /* type Qiniu_Count */ diff --git a/qiniu/base.h b/qiniu/base.h index e57cac6..c53415f 100644 --- a/qiniu/base.h +++ b/qiniu/base.h @@ -110,6 +110,7 @@ typedef unsigned long long Qiniu_Uint64; QINIU_DLLAPI extern void Qiniu_Free(void *addr); QINIU_DLLAPI extern void Qiniu_FreeV2(void **addr); + QINIU_DLLAPI extern void Qiniu_Multi_Free(int n, ...); /*============================================================================*/ /* type Qiniu_Count */ diff --git a/qiniu/code.c b/qiniu/code.c new file mode 100644 index 0000000..3288c96 --- /dev/null +++ b/qiniu/code.c @@ -0,0 +1,42 @@ +#include +#include "private/code.h" + +Qiniu_Retry_Decision _Qiniu_Should_Retry(int code) +{ + if (code / 100 == 4 && code != 406 && code != 429) + { + return QINIU_DONT_RETRY; + } + switch (code) + { + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_CONNECT: + case CURLE_WEIRD_SERVER_REPLY: + case CURLE_PARTIAL_FILE: + case CURLE_UPLOAD_FAILED: + case CURLE_OPERATION_TIMEDOUT: + case CURLE_SSL_CONNECT_ERROR: + case CURLE_SEND_ERROR: + case CURLE_RECV_ERROR: + case CURLE_SSL_CERTPROBLEM: + case CURLE_SSL_CIPHER: + case CURLE_PEER_FAILED_VERIFICATION: + case CURLE_HTTP2_STREAM: + case CURLE_HTTP3: + case CURLE_QUIC_CONNECT_ERROR: + case 406: + case 429: + case 500: + case 502: + case 503: + case 504: + case 509: + case 571: + case 573: + case 599: + return QINIU_TRY_NEXT_DOMAIN; + + default: + return QINIU_DONT_RETRY; + } +} diff --git a/qiniu/http.c b/qiniu/http.c index a43812f..c82372a 100644 --- a/qiniu/http.c +++ b/qiniu/http.c @@ -348,6 +348,7 @@ void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize) Qiniu_Zero_Ptr(self); self->curl = curl_easy_init(); self->auth = auth; + self->retriesMax = SIZE_MAX; Qiniu_Buffer_Init(&self->b, bufSize); Qiniu_Buffer_Init(&self->respHeader, bufSize); @@ -396,6 +397,11 @@ void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long self->lowSpeedTime = lowSpeedTime; } // Qiniu_Client_SetLowSpeedLimit +void Qiniu_Client_SetMaximumRetries(Qiniu_Client *self, size_t retriesMax) +{ + self->retriesMax = retriesMax; +} // Qiniu_Client_SetMaximumRetries + void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs) { self->timeoutMs = timeoutMs; @@ -475,7 +481,7 @@ static Qiniu_Error Qiniu_Client_callWithBody( Qiniu_Error err; const char *ctxType; char ctxLength[64], userAgent[64]; - Qiniu_Header *headers = NULL; + Qiniu_Header *headers; CURL *curl = (CURL *)self->curl; err = Qiniu_Client_config(self); if (err.code != Qiniu_OK.code) diff --git a/qiniu/http.h b/qiniu/http.h index a06a5bc..3b48b2c 100644 --- a/qiniu/http.h +++ b/qiniu/http.h @@ -139,6 +139,9 @@ extern "C" // Millisecond timeout for the connection phase. long connectTimeoutMs; + + // Max retries count. + size_t retriesMax; }; typedef struct _Qiniu_Client Qiniu_Client; @@ -146,6 +149,7 @@ extern "C" QINIU_DLLAPI extern void Qiniu_Client_Cleanup(Qiniu_Client *self); QINIU_DLLAPI extern void Qiniu_Client_BindNic(Qiniu_Client *self, const char *nic); QINIU_DLLAPI extern void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long lowSpeedTime); + QINIU_DLLAPI extern void Qiniu_Client_SetMaximumRetries(Qiniu_Client *self, size_t retries); QINIU_DLLAPI extern void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs); QINIU_DLLAPI extern void Qiniu_Client_SetConnectTimeout(Qiniu_Client *self, long connectTimeoutMs); QINIU_DLLAPI extern void Qiniu_Client_EnableAutoQuery(Qiniu_Client *self, Qiniu_Bool useHttps); diff --git a/qiniu/io.c b/qiniu/io.c index d030d05..fb752b9 100644 --- a/qiniu/io.c +++ b/qiniu/io.c @@ -11,6 +11,7 @@ #include "reader.h" #include "recorder_utils.h" #include "private/region.h" +#include "private/code.h" #include /*============================================================================*/ @@ -58,20 +59,27 @@ CURL *Qiniu_Client_reset(Qiniu_Client *self); Qiniu_Error Qiniu_callex(CURL *curl, Qiniu_Buffer *resp, Qiniu_Json **ret, Qiniu_Bool simpleError, Qiniu_Buffer *resph); -static Qiniu_Error Get_Qiniu_UpHost(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char **upHost) +static Qiniu_Error Get_Qiniu_UpHosts(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char *const **upHosts, size_t *upHostsCount) { if (extra && extra->ipCount != 0) { Qiniu_Count oldIndex = Qiniu_Count_Inc(&extra->ipIndex); - *upHost = extra->upIps[abs(oldIndex % extra->ipCount)]; + *upHosts = &extra->upIps[labs(oldIndex % extra->ipCount)]; + *upHostsCount = 1; } else if (extra && extra->upHost != NULL) { - *upHost = extra->upHost; + *upHosts = &extra->upHost; + *upHostsCount = 1; + } + else if (extra && extra->upHosts != NULL && extra->upHostsCount != 0) + { + *upHosts = extra->upHosts; + *upHostsCount = extra->upHostsCount; } else { - Qiniu_Error err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, upHost); + Qiniu_Error err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, upHosts, upHostsCount); if (err.code != Qiniu_OK.code) { return err; @@ -150,48 +158,61 @@ static Qiniu_Error Qiniu_Io_call( { int retCode = 0; Qiniu_Error err; - struct curl_slist *headers = NULL; - const char *upHost = NULL; + struct curl_slist *headers; + const char *const *upHosts; + const char *defaultUpHosts[] = {QINIU_UP_HOST}; + size_t upHostsCount; - err = Qiniu_Client_config(self); + headers = curl_slist_append(NULL, "Expect:"); + err = Get_Qiniu_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount); if (err.code != Qiniu_OK.code) { return err; } - err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost); - if (err.code != Qiniu_OK.code) + if (upHostsCount == 0) { - return err; + upHosts = defaultUpHosts; + upHostsCount = 1; } - headers = curl_slist_append(NULL, "Expect:"); - - CURL *curl = Qiniu_Client_reset(self); - curl_easy_setopt(curl, CURLOPT_URL, upHost); - curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - //// For aborting uploading file. - if (extra->upAbortCallback) + for (size_t retries = 0; retries < upHostsCount && retries <= self->retriesMax; retries++) { - curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback); - } // if + CURL *curl = Qiniu_Client_reset(self); + err = Qiniu_Client_config(self); + if (err.code != Qiniu_OK.code) + { + return err; + } + curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries]); + curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader); - if (err.code == Qiniu_OK.code && ret != NULL) - { - if (extra->callbackRetParser != NULL) + //// For aborting uploading file. + if (extra->upAbortCallback) { - err = (*extra->callbackRetParser)(extra->callbackRet, self->root); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback); + } // if + + err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader); + if (err.code == Qiniu_OK.code) + { + if (extra->callbackRetParser != NULL) + { + err = (*extra->callbackRetParser)(extra->callbackRet, self->root); + } + else if (ret != NULL) + { + ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL); + ret->key = Qiniu_Json_GetString(self->root, "key", NULL); + ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL); + } + break; } - else + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) { - ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL); - ret->key = Qiniu_Json_GetString(self->root, "key", NULL); - ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL); + break; } } - curl_formfree(formpost); curl_slist_free_all(headers); return err; @@ -327,42 +348,56 @@ static Qiniu_Error Qiniu_Io_call_with_callback( { int retCode = 0; Qiniu_Error err; - struct curl_slist *headers = NULL; - const char *upHost; + struct curl_slist *headers; + const char *const *upHosts; + const char *defaultUpHosts[] = {QINIU_UP_HOST}; + size_t upHostsCount; - CURL *curl = Qiniu_Client_reset(self); - err = Qiniu_Client_config(self); + headers = curl_slist_append(NULL, "Expect:"); + err = Get_Qiniu_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount); if (err.code != Qiniu_OK.code) { return err; } - err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost); - if (err.code != Qiniu_OK.code) + if (upHostsCount == 0) { - return err; + upHosts = defaultUpHosts; + upHostsCount = 1; } - headers = curl_slist_append(NULL, "Expect:"); + for (size_t retries = 0; retries < upHostsCount && retries <= self->retriesMax; retries++) + { + CURL *curl = Qiniu_Client_reset(self); + err = Qiniu_Client_config(self); + if (err.code != Qiniu_OK.code) + { + return err; + } - curl_easy_setopt(curl, CURLOPT_URL, upHost); - curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr); + curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries]); + curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr); - err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader); - if (err.code == Qiniu_OK.code && ret != NULL) - { - if (extra->callbackRetParser != NULL) + err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader); + if (err.code == Qiniu_OK.code) { - err = (*extra->callbackRetParser)(extra->callbackRet, self->root); + if (extra->callbackRetParser != NULL) + { + err = (*extra->callbackRetParser)(extra->callbackRet, self->root); + } + else if (ret != NULL) + { + ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL); + ret->key = Qiniu_Json_GetString(self->root, "key", NULL); + } + break; } - else + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) { - ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL); - ret->key = Qiniu_Json_GetString(self->root, "key", NULL); + break; } } - curl_formfree(formpost); curl_slist_free_all(headers); return err; diff --git a/qiniu/io.h b/qiniu/io.h index 2329bca..25a1fb9 100644 --- a/qiniu/io.h +++ b/qiniu/io.h @@ -3,7 +3,7 @@ Name : io.h Author : Qiniu.com Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd. - Description : + Description : ============================================================================ */ @@ -51,9 +51,14 @@ extern "C" Qiniu_Rd_FnAbort upAbortCallback; const char *upHost; + const char **upIps; Qiniu_Count ipCount; Qiniu_Count ipIndex; + + // Specify multiple upHosts + const char *const *upHosts; + size_t upHostsCount; } Qiniu_Io_PutExtra; /*============================================================================*/ diff --git a/qiniu/multipart_upload.c b/qiniu/multipart_upload.c index e2d5cc8..f1aff85 100644 --- a/qiniu/multipart_upload.c +++ b/qiniu/multipart_upload.c @@ -16,13 +16,13 @@ #include "../cJSON/cJSON.h" #include "tm.h" #include "private/region.h" +#include "private/code.h" /*============================================================================*/ #if defined(_WIN32) #include #endif -static void Qiniu_Multi_Free(int n, ...); static char *encodeKey(const char *key, bool hasKey); static void restoreAuth(Qiniu_Auth *a, Qiniu_Auth backup); static cJSON *buildJsonMap(int kvNum, const char *(*kvpairs)[2]); @@ -124,67 +124,94 @@ getTotalPartNum(Qiniu_Int64 fileSize, Qiniu_Int64 partSize, int *totalPartNum) static Qiniu_Error init_upload(Qiniu_Client *client, const char *bucket, const char *encodedKey, Qiniu_Multipart_PutExtra *extraParam, Qiniu_InitPart_Ret *initPartRet) { Qiniu_Error err = Qiniu_OK; - const char *uphost = extraParam->upHost; - char *reqUrl = Qiniu_String_Concat(uphost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads", NULL); + const char *const *upHosts; + size_t upHostsCount; - for (int i = 0; i < extraParam->tryTimes; i++) + if (extraParam->upHost != NULL) { + upHosts = &extraParam->upHost; + upHostsCount = 1; + } + else + { + upHosts = extraParam->upHosts; + upHostsCount = extraParam->upHostsCount; + } + + for (int tries = 0; tries < extraParam->tryTimes && tries <= client->retriesMax; tries++) + { + const char *upHost = upHosts[tries % upHostsCount]; + const char *reqUrl = Qiniu_String_Concat(upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads", NULL); Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call. err = Qiniu_Client_Call(client, &callRet, reqUrl); - if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx , net error will retry + Qiniu_Free((void *)reqUrl); + if (err.code == Qiniu_OK.code) { - Qiniu_Log_Error("init_upload retry: %E", err); - continue; + if (initPartRet != NULL) + { + initPartRet->uploadId = Qiniu_String_Dup(Qiniu_Json_GetString(callRet, "uploadId", NULL)); + initPartRet->expireAt = Qiniu_Json_GetUInt64(callRet, "expireAt", 0); + break; + } } - else if (err.code != Qiniu_OK.code) + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) { - Qiniu_Log_Error("init_upload: %E", err); break; } - - initPartRet->uploadId = Qiniu_String_Dup(Qiniu_Json_GetString(callRet, "uploadId", NULL)); - initPartRet->expireAt = Qiniu_Json_GetUInt64(callRet, "expireAt", 0); - break; } - Qiniu_Free(reqUrl); return err; } -Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extraParam, const char *reqUrl, int partNum, Qiniu_ReaderAt reader, Qiniu_Int64 partOffset, Qiniu_Int64 partSize, const char *md5str, Qiniu_UploadPartResp *ret) +Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extraParam, const char *path, int partNum, Qiniu_ReaderAt reader, Qiniu_Int64 partOffset, Qiniu_Int64 partSize, const char *md5str, Qiniu_UploadPartResp *ret) { Qiniu_Error err = Qiniu_OK; - for (int try = 0; try < extraParam->tryTimes; try++) + const char *const *upHosts; + size_t upHostsCount; + + if (extraParam->upHost != NULL) + { + upHosts = &extraParam->upHost; + upHostsCount = 1; + } + else + { + upHosts = extraParam->upHosts; + upHostsCount = extraParam->upHostsCount; + } + for (int tries = 0; tries < extraParam->tryTimes && tries <= client->retriesMax; tries++) { Qiniu_Section section; Qiniu_Zero(section); Qiniu_Reader thisPartBody = Qiniu_SectionReader(§ion, reader, (Qiniu_Off_T)partOffset, partSize); Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call. + const char *upHost = upHosts[tries % upHostsCount]; + const char *reqUrl = Qiniu_String_Concat(upHost, path, NULL); err = Qiniu_Client_CallWithMethod(client, &callRet, reqUrl, thisPartBody, partSize, NULL, "PUT", md5str); - if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx,net error will retry + Qiniu_Free((void *)reqUrl); + if (err.code == Qiniu_OK.code) { - Qiniu_Log_Error("upload_part retry: partNum:%d, %E", partNum, err); - continue; + if (ret != NULL) + { + const char *md5 = Qiniu_Json_GetString(callRet, "md5", NULL); + const char *etag = Qiniu_Json_GetString(callRet, "etag", NULL); + ret->etag = Qiniu_String_Dup(etag); + ret->md5 = Qiniu_String_Dup(md5); + ret->partNum = partNum; + break; + } } - else if (err.code != Qiniu_OK.code) + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) { - Qiniu_Log_Error("upload_part: partNum:%d, %E", partNum, err); break; } - - const char *md5 = Qiniu_Json_GetString(callRet, "md5", NULL); - const char *etag = Qiniu_Json_GetString(callRet, "etag", NULL); - ret->etag = Qiniu_String_Dup(etag); - ret->md5 = Qiniu_String_Dup(md5); - ret->partNum = partNum; - // Qiniu_Log_Debug("partNum:%d,remote md5:%s ", partNum, md5); - break; } + // notify callback - if ((err.code == Qiniu_OK.code) && extraParam->notify) + if (err.code == Qiniu_OK.code && extraParam->notify) { extraParam->notify(ret); } - if ((err.code != Qiniu_OK.code) && extraParam->notifyErr) + else if (err.code != Qiniu_OK.code && extraParam->notifyErr) { extraParam->notifyErr(partNum, err); } @@ -195,10 +222,9 @@ Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extr static Qiniu_Error upload_parts(Qiniu_Client *client, const char *bucket, const char *encodedKey, const Qiniu_InitPart_Ret *initParts, Qiniu_ReaderAt *reader, Qiniu_Int64 fsize, int totalPartNum, Qiniu_Multipart_PutExtra *extraParam, Qiniu_Multipart_Recorder *recorder, Qiniu_UploadParts_Ret *uploadPartsRet) { Qiniu_Error err = Qiniu_OK; - const char *uphost = extraParam->upHost; - Qiniu_Int64 partSize = extraParam->partSize; - int lastPart = totalPartNum - 1; + const int lastPart = totalPartNum - 1; + for (int partNum = 0; partNum < totalPartNum; partNum++) { if ((uploadPartsRet->PartsRet + partNum)->etag != NULL) @@ -206,12 +232,12 @@ static Qiniu_Error upload_parts(Qiniu_Client *client, const char *bucket, const continue; } - int partNumInReq = partNum + 1; // partNum start from 1 - char partNumStr[10]; // valid partNum ={"1"~"10000"} + const int partNumInReq = partNum + 1; // partNum start from 1 + char partNumStr[10]; // valid partNum ={"1"~"10000"} snprintf(partNumStr, 10, "%d", partNumInReq); - char *reqUrl = Qiniu_String_Concat(uphost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initParts->uploadId, "/", partNumStr, NULL); + const char *path = Qiniu_String_Concat("/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initParts->uploadId, "/", partNumStr, NULL); - Qiniu_Int64 thisPartOffset = partNum * partSize; + const Qiniu_Int64 thisPartOffset = partNum * partSize; Qiniu_Int64 thisPartSize = partSize; if (partNum == lastPart) { @@ -224,9 +250,9 @@ static Qiniu_Error upload_parts(Qiniu_Client *client, const char *bucket, const md5str = caculatePartMd5(*reader, thisPartOffset, thisPartSize); // Qiniu_Log_Debug("partNum:%d, local Md5:%s ", partNumInReq, md5str); } - err = upload_one_part(client, extraParam, reqUrl, partNumInReq, *reader, thisPartOffset, thisPartSize, md5str, &uploadPartsRet->PartsRet[partNum]); + err = upload_one_part(client, extraParam, path, partNumInReq, *reader, thisPartOffset, thisPartSize, md5str, &uploadPartsRet->PartsRet[partNum]); - Qiniu_Multi_Free(2, (void *)reqUrl, (void *)md5str); + Qiniu_Multi_Free(2, (void *)path, (void *)md5str); if (err.code != Qiniu_OK.code) { @@ -283,35 +309,53 @@ static Qiniu_Error complete_upload( // step2:send req Qiniu_Error err = Qiniu_OK; - char *reqUrl = Qiniu_String_Concat(extraParam->upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initPartsRet->uploadId, NULL); + const char *const *upHosts; + size_t upHostsCount; + + if (extraParam->upHost != NULL) + { + upHosts = &extraParam->upHost; + upHostsCount = 1; + } + else + { + upHosts = extraParam->upHosts; + upHostsCount = extraParam->upHostsCount; + } - for (int try = 0; try < extraParam->tryTimes; try++) + for (int tries = 0; tries < extraParam->tryTimes && tries <= client->retriesMax; tries++) { + const char *upHost = upHosts[tries % upHostsCount]; + char *reqUrl = Qiniu_String_Concat(upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initPartsRet->uploadId, NULL); Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call. err = Qiniu_Client_CallWithBuffer(client, &callRet, reqUrl, body, strlen(body), "application/json"); - if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx,net error will retry + Qiniu_Free((void *)reqUrl); + if (err.code == Qiniu_OK.code) { - Qiniu_Log_Error("cupload: retry uploadId:%s, %E", initPartsRet->uploadId, err); - continue; - } - if (err.code / 100 != 4 && extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL) - { - extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey); + if (extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL) + { + extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey); + } + if (callRet != NULL) + { + const char *hash = Qiniu_Json_GetString(callRet, "hash", NULL); // don't free(hash) since no malloc happen. + const char *key = Qiniu_Json_GetString(callRet, "key", NULL); + completeRet->hash = Qiniu_String_Dup(hash); + completeRet->key = Qiniu_String_Dup(key); + break; + } } - if (err.code != Qiniu_OK.code) + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) { - Qiniu_Log_Error("cupload: uploadId:%s, %E", initPartsRet->uploadId, err); + if (err.code / 100 != 4 && extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL) + { + extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey); + } break; } - - const char *hash = Qiniu_Json_GetString(callRet, "hash", NULL); // don't free(hash) since no malloc happen. - const char *key = Qiniu_Json_GetString(callRet, "key", NULL); - completeRet->hash = Qiniu_String_Dup(hash); - completeRet->key = Qiniu_String_Dup(key); - break; } // Qiniu_Log_Debug("Upload result: uploadid:%s, hash:%s, key:%s ", uploadId, completeRet->hash, completeRet->key); - Qiniu_Multi_Free(2, (void *)reqUrl, (void *)body); + Qiniu_Free((void *)body); return err; } @@ -506,9 +550,9 @@ Qiniu_Error verifyParam(Qiniu_Client *client, const char *accessKey, const char { param->tryTimes = 3; } - if (param->upHost == NULL) + if (param->upHost == NULL && (param->upHosts == NULL || param->upHostsCount == 0)) { - err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, ¶m->upHost); + err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, ¶m->upHosts, ¶m->upHostsCount); } return err; @@ -542,19 +586,6 @@ cJSON *buildJsonMap(int kvNum, const char *(*kvpairs)[2]) } return map; } -//-----------------------------------memory free helper func -void Qiniu_Multi_Free(int n, ...) -{ - void *p = NULL; - va_list v1; - va_start(v1, n); - for (int i = 0; i < n; i++) - { - p = va_arg(v1, void *); - free(p); - } - va_end(v1); -} Qiniu_Error readMedium(struct Qiniu_Record_Medium *medium, char **uploadId, Qiniu_Uint64 *expireAt, Qiniu_UploadPartResp *ret) { diff --git a/qiniu/multipart_upload.h b/qiniu/multipart_upload.h index 3d265ca..cab7297 100644 --- a/qiniu/multipart_upload.h +++ b/qiniu/multipart_upload.h @@ -29,7 +29,7 @@ extern "C" { char *md5; char *etag; - int partNum; //Attention: partNum start with 1 + int partNum; // Attention: partNum start with 1 } Qiniu_UploadPartResp; typedef void (*NotifyFunc)(Qiniu_UploadPartResp *partResp); @@ -37,11 +37,11 @@ extern "C" typedef struct _Qiniu_Multipart_PutExtra { - const char *upHost; //if not set explicitly ,will use global QINIU_UP_HOST; - Qiniu_Int64 partSize; //size for each part + const char *upHost; // if not set explicitly, will use upHosts; + Qiniu_Int64 partSize; // size for each part const char *mimeType; - int tryTimes; //at least 1, default=3 - Qiniu_Bool enableContentMd5; //calulate md5 and set to request.header["Content-MD5"] + int tryTimes; // at least 1, default=3 + Qiniu_Bool enableContentMd5; // calulate md5 and set to request.header["Content-MD5"] NotifyFunc notify; NotifyErrFunc notifyErr; @@ -58,6 +58,9 @@ extern "C" Qiniu_Recorder *recorder; + // Specify multiple upHosts, if not set explicitly, will global QINIU_UP_HOST + const char *const *upHosts; + size_t upHostsCount; } Qiniu_Multipart_PutExtra; typedef struct diff --git a/qiniu/private/code.h b/qiniu/private/code.h new file mode 100644 index 0000000..44417ac --- /dev/null +++ b/qiniu/private/code.h @@ -0,0 +1,7 @@ +typedef enum +{ + QINIU_DONT_RETRY, + QINIU_TRY_NEXT_DOMAIN +} Qiniu_Retry_Decision; + +Qiniu_Retry_Decision _Qiniu_Should_Retry(int code); diff --git a/qiniu/private/region.h b/qiniu/private/region.h index 8a5cc46..7075d76 100644 --- a/qiniu/private/region.h +++ b/qiniu/private/region.h @@ -2,15 +2,13 @@ #define QINIU_PRIVATE_REGION_H #include #include "../base.h" +#include "../http.h" #ifdef __cplusplus extern "C" { #endif - struct _Qiniu_Client; - typedef struct _Qiniu_Client Qiniu_Client; - struct _Qiniu_Region_Hosts { const char *const *hosts; @@ -45,10 +43,15 @@ extern "C" static void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service); Qiniu_Error _Qiniu_Region_Get_Up_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); + Qiniu_Error _Qiniu_Region_Get_Up_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); + Qiniu_Error _Qiniu_Region_Get_Io_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); + Qiniu_Error _Qiniu_Region_Get_Rs_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); + Qiniu_Error _Qiniu_Region_Get_Rsf_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); + Qiniu_Error _Qiniu_Region_Get_Api_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); Qiniu_Error _Qiniu_Region_Query(Qiniu_Client *self, struct _Qiniu_Region **region, const char *accessKey, const char *const bucketName, Qiniu_Bool useHttps); #ifdef __cplusplus diff --git a/qiniu/recorder.c b/qiniu/recorder.c index e7128d0..7f8b5e5 100644 --- a/qiniu/recorder.c +++ b/qiniu/recorder.c @@ -157,7 +157,6 @@ Qiniu_Error _Qiniu_FileSystem_RecordMedium_Read_Entry(const struct Qiniu_Record_ Qiniu_Error _Qiniu_FileSystem_RecordMedium_Has_Next_Entry(const struct Qiniu_Record_Medium *medium, Qiniu_Bool *has) { struct _FileSystem_Recorder_Data *data = (struct _FileSystem_Recorder_Data *)medium->data; - size_t i; Qiniu_Bool hasEntry = Qiniu_True; if (data->buf == NULL || data->bufOffset >= data->bufSize) diff --git a/qiniu/region.c b/qiniu/region.c index 94bd93f..50c3706 100644 --- a/qiniu/region.c +++ b/qiniu/region.c @@ -1371,6 +1371,24 @@ Qiniu_Error _Qiniu_Region_Get_Up_Host(Qiniu_Client *self, const char *accessKey, { const char *const *hosts; size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Up_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != Qiniu_OK.code) + { + return err; + } + if (count == 0) + { + *host = QINIU_UP_HOST; + } + else + { + *host = hosts[0]; + } + return err; +} + +Qiniu_Error _Qiniu_Region_Get_Up_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) +{ Qiniu_Error err = Qiniu_OK; Qiniu_Region *foundRegion = NULL; @@ -1386,14 +1404,25 @@ Qiniu_Error _Qiniu_Region_Get_Up_Host(Qiniu_Client *self, const char *accessKey, { goto foundCache; } - *host = QINIU_UP_HOST; return err; } foundCache: - hosts = Qiniu_Region_Get_Up_Preferred_Hosts(foundRegion, &count); + *hosts = Qiniu_Region_Get_Up_Preferred_Hosts(foundRegion, count); + return err; +} + +Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +{ + const char *const *hosts; + size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Io_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != Qiniu_OK.code) + { + return err; + } if (count == 0) { - *host = QINIU_UP_HOST; + *host = QINIU_IOVIP_HOST; } else { @@ -1402,10 +1431,8 @@ Qiniu_Error _Qiniu_Region_Get_Up_Host(Qiniu_Client *self, const char *accessKey, return err; } -Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +Qiniu_Error _Qiniu_Region_Get_Io_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) { - const char *const *hosts; - size_t count; Qiniu_Error err = Qiniu_OK; Qiniu_Region *foundRegion = NULL; @@ -1421,14 +1448,25 @@ Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, { goto foundCache; } - *host = QINIU_UP_HOST; return err; } foundCache: - hosts = Qiniu_Region_Get_Io_Preferred_Hosts(foundRegion, &count); + *hosts = Qiniu_Region_Get_Io_Preferred_Hosts(foundRegion, count); + return err; +} + +Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +{ + const char *const *hosts; + size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Rs_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != Qiniu_OK.code) + { + return err; + } if (count == 0) { - *host = QINIU_IOVIP_HOST; + *host = QINIU_RS_HOST; } else { @@ -1437,10 +1475,8 @@ Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, return err; } -Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +Qiniu_Error _Qiniu_Region_Get_Rs_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) { - const char *const *hosts; - size_t count; Qiniu_Error err = Qiniu_OK; Qiniu_Region *foundRegion = NULL; @@ -1456,14 +1492,25 @@ Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, { goto foundCache; } - *host = QINIU_UP_HOST; return err; } foundCache: - hosts = Qiniu_Region_Get_Rs_Preferred_Hosts(foundRegion, &count); + *hosts = Qiniu_Region_Get_Rs_Preferred_Hosts(foundRegion, count); + return err; +} + +Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +{ + const char *const *hosts; + size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Rsf_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != Qiniu_OK.code) + { + return err; + } if (count == 0) { - *host = QINIU_RS_HOST; + *host = QINIU_RSF_HOST; } else { @@ -1472,10 +1519,8 @@ Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, return err; } -Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +Qiniu_Error _Qiniu_Region_Get_Rsf_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) { - const char *const *hosts; - size_t count; Qiniu_Error err = Qiniu_OK; Qiniu_Region *foundRegion = NULL; @@ -1491,14 +1536,25 @@ Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey { goto foundCache; } - *host = QINIU_UP_HOST; return err; } foundCache: - hosts = Qiniu_Region_Get_Rsf_Preferred_Hosts(foundRegion, &count); + *hosts = Qiniu_Region_Get_Rsf_Preferred_Hosts(foundRegion, count); + return err; +} + +Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +{ + const char *const *hosts; + size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Api_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != Qiniu_OK.code) + { + return err; + } if (count == 0) { - *host = QINIU_RSF_HOST; + *host = QINIU_API_HOST; } else { @@ -1507,10 +1563,8 @@ Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey return err; } -Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +Qiniu_Error _Qiniu_Region_Get_Api_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) { - const char *const *hosts; - size_t count; Qiniu_Error err = Qiniu_OK; Qiniu_Region *foundRegion = NULL; @@ -1526,18 +1580,9 @@ Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey { goto foundCache; } - *host = QINIU_UP_HOST; return err; } foundCache: - hosts = Qiniu_Region_Get_Api_Preferred_Hosts(foundRegion, &count); - if (count == 0) - { - *host = QINIU_API_HOST; - } - else - { - *host = hosts[0]; - } + *hosts = Qiniu_Region_Get_Api_Preferred_Hosts(foundRegion, count); return err; } diff --git a/qiniu/region.h b/qiniu/region.h index 9d3d019..1d08a24 100644 --- a/qiniu/region.h +++ b/qiniu/region.h @@ -16,12 +16,6 @@ extern "C" { #endif - struct _Qiniu_Client; - typedef struct _Qiniu_Client Qiniu_Client; - - struct _Qiniu_Region; - typedef struct _Qiniu_Region Qiniu_Region; - QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, size_t *count); QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, size_t *count); QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, size_t *count); diff --git a/qiniu/resumable_io.c b/qiniu/resumable_io.c index e988b1d..eca7716 100644 --- a/qiniu/resumable_io.c +++ b/qiniu/resumable_io.c @@ -17,6 +17,7 @@ #include "recorder_utils.h" #include "../cJSON/cJSON.h" #include "private/region.h" +#include "private/code.h" #define blockBits 22 #define blockMask ((1 << blockBits) - 1) @@ -134,10 +135,13 @@ static int Qiniu_Rio_ST_RunTask(void *self, void (*task)(void *params), void *pa static Qiniu_Rio_ThreadModel_Itbl Qiniu_Rio_ST_Itbl = { Qiniu_Rio_ST_WaitGroup, Qiniu_Rio_ST_ClientTls, - Qiniu_Rio_ST_RunTask}; + Qiniu_Rio_ST_RunTask, +}; Qiniu_Rio_ThreadModel Qiniu_Rio_ST = { - NULL, &Qiniu_Rio_ST_Itbl}; + NULL, + &Qiniu_Rio_ST_Itbl, +}; /*============================================================================*/ /* type Qiniu_Rio_Settings */ @@ -147,7 +151,11 @@ static Qiniu_Rio_Settings settings = { defaultWorkers, defaultChunkSize, defaultTryTimes, - {NULL, &Qiniu_Rio_ST_Itbl}}; + { + NULL, + &Qiniu_Rio_ST_Itbl, + }, +}; void Qiniu_Rio_SetSettings(Qiniu_Rio_Settings *v) { @@ -385,12 +393,12 @@ static Qiniu_Error Qiniu_Rio_bput( } static Qiniu_Error Qiniu_Rio_Mkblock( - Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, int blkSize, Qiniu_Reader body, int bodyLength, - Qiniu_Rio_PutExtra *extra) + Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, int blkSize, const char *upHost, + Qiniu_Reader body, int bodyLength, Qiniu_Rio_PutExtra *extra) { - char *url = Qiniu_String_Format(128, "%s/mkblk/%d", extra->upHost, blkSize); + const char *url = Qiniu_String_Format(128, "%s/mkblk/%d", upHost, blkSize); Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url); - Qiniu_Free(url); + Qiniu_Free((void *)url); return err; } @@ -398,9 +406,10 @@ static Qiniu_Error Qiniu_Rio_Mkblock( static Qiniu_Error Qiniu_Rio_Blockput( Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, Qiniu_Reader body, int bodyLength) { - char *url = Qiniu_String_Format(1024, "%s/bput/%s/%d", ret->host, ret->ctx, (int)ret->offset); + const char *url = Qiniu_String_Format(1024, "%s/bput/%s/%d", ret->host, ret->ctx, (int)ret->offset); Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url); - Qiniu_Free(url); + Qiniu_Free((void *)url); + return err; } @@ -409,19 +418,16 @@ static Qiniu_Error Qiniu_Rio_Blockput( static Qiniu_Error ErrUnmatchedChecksum = { Qiniu_Rio_UnmatchedChecksum, "unmatched checksum"}; -static int Qiniu_TemporaryError(int code) -{ - return code / 100 != 4; -} - static Qiniu_Error Qiniu_Rio_ResumableBlockput( Qiniu_Client *c, Qiniu_Rio_BlkputRet *ret, Qiniu_ReaderAt f, int blkIdx, int blkSize, - Qiniu_Rio_PutExtra *extra, size_t *chunksUploaded) + Qiniu_Rio_PutExtra *extra, size_t *pChunksUploaded) { Qiniu_Error err = Qiniu_OK; Qiniu_Tee tee; Qiniu_Section section; Qiniu_Reader body, body1; + const char *const *upHosts; + size_t upHostsCount; Qiniu_Crc32 crc32; Qiniu_Writer h = Qiniu_Crc32Writer(&crc32, 0); @@ -429,13 +435,22 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( int chunkSize = extra->chunkSize; int bodyLength; - int tryTimes; int notifyRet = 0; - size_t httpCalled = 0; + size_t chunksUploaded = 0; - if (ret->ctx == NULL) + if (extra->upHost != NULL) + { + upHosts = &extra->upHost; + upHostsCount = 1; + } + else { + upHosts = extra->upHosts; + upHostsCount = extra->upHostsCount; + } + if (ret->ctx == NULL) + { if (chunkSize < blkSize) { bodyLength = chunkSize; @@ -445,16 +460,23 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( bodyLength = blkSize; } - body1 = Qiniu_SectionReader(§ion, f, (Qiniu_Off_T)offbase, bodyLength); - body = Qiniu_TeeReader(&tee, body1, h); - - err = Qiniu_Rio_Mkblock(c, ret, blkSize, body, bodyLength, extra); - if (err.code != Qiniu_OK.code) + for (int tries = 0; tries < extra->tryTimes && tries <= c->retriesMax; tries++) { - return err; - } + body1 = Qiniu_SectionReader(§ion, f, (Qiniu_Off_T)offbase, bodyLength); + body = Qiniu_TeeReader(&tee, body1, h); - httpCalled++; + const char *upHost = upHosts[tries % upHostsCount]; + err = Qiniu_Rio_Mkblock(c, ret, blkSize, upHost, body, bodyLength, extra); + if (err.code == Qiniu_OK.code) + { + break; + } + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) + { + return err; + } + } + chunksUploaded++; if (ret->crc32 != crc32.val || (int)(ret->offset) != bodyLength) { return ErrUnmatchedChecksum; @@ -481,7 +503,11 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( bodyLength = blkSize - (int)(ret->offset); } - tryTimes = extra->tryTimes; + int tryTimes = extra->tryTimes; + if (tryTimes > c->retriesMax + 1) + { + tryTimes = c->retriesMax + 1; + } lzRetry: crc32.val = 0; @@ -491,7 +517,7 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( err = Qiniu_Rio_Blockput(c, ret, body, bodyLength); if (err.code == Qiniu_OK.code) { - httpCalled++; + chunksUploaded++; if (ret->crc32 == crc32.val) { notifyRet = extra->notify(extra->notifyRecvr, blkIdx, blkSize, ret); @@ -518,7 +544,7 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( } Qiniu_Log_Warn("ResumableBlockput %d off:%d failed - %E", blkIdx, (int)ret->offset, err); } - if (tryTimes > 1 && Qiniu_TemporaryError(err.code)) + if (tryTimes > 1 && _Qiniu_Should_Retry(err.code) != QINIU_DONT_RETRY) { tryTimes--; Qiniu_Log_Info("ResumableBlockput %E, retrying ...", err); @@ -527,9 +553,9 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( break; } - if (chunksUploaded != NULL) + if (pChunksUploaded != NULL) { - *chunksUploaded = httpCalled; + *pChunksUploaded = chunksUploaded; } return err; @@ -538,7 +564,7 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( /*============================================================================*/ static Qiniu_Error Qiniu_Rio_Mkfile( - Qiniu_Client *c, Qiniu_Rio_PutRet *ret, const char *key, Qiniu_Int64 fsize, Qiniu_Rio_PutExtra *extra) + Qiniu_Client *c, Qiniu_Rio_PutRet *ret, const char *upHost, const char *key, Qiniu_Int64 fsize, Qiniu_Rio_PutExtra *extra) { size_t i, blkCount = extra->blockCnt; Qiniu_Json *root; @@ -547,7 +573,7 @@ static Qiniu_Error Qiniu_Rio_Mkfile( int j = 0; Qiniu_Buffer_Init(&url, 2048); - Qiniu_Buffer_AppendFormat(&url, "%s/mkfile/%D", extra->upHost, fsize); + Qiniu_Buffer_AppendFormat(&url, "%s/mkfile/%D", upHost, fsize); if (key != NULL) { @@ -714,7 +740,7 @@ static void Qiniu_Rio_doTask(void *params) return; } - if (tryTimes > 1 && Qiniu_TemporaryError(err.code)) + if (tryTimes > 1 && _Qiniu_Should_Retry(err.code) != QINIU_DONT_RETRY) { tryTimes--; Qiniu_Log_Info("resumable.Put %E, retrying ...", err); @@ -812,7 +838,7 @@ static Qiniu_Error _Qiniu_Rio_Put( { return err; } - if (extra.upHost == NULL) + if (extra.upHost == NULL && (extra.upHosts == NULL || extra.upHostsCount == 0)) { if (!Qiniu_Utils_Extract_Bucket(uptoken, &accessKey, &bucketName)) { @@ -820,11 +846,10 @@ static Qiniu_Error _Qiniu_Rio_Put( err.message = "parse uptoken failed"; return err; } - err = _Qiniu_Region_Get_Up_Host(self, accessKey, bucketName, &extra.upHost); + err = _Qiniu_Region_Get_Up_Hosts(self, accessKey, bucketName, &extra.upHosts, &extra.upHostsCount); if (err.code != Qiniu_OK.code) { - Qiniu_Free((void *)accessKey); - Qiniu_Free((void *)bucketName); + Qiniu_Multi_Free(2, (void *)accessKey, (void *)bucketName); return err; } } @@ -865,7 +890,7 @@ static Qiniu_Error _Qiniu_Rio_Put( { wg.itbl->Done(wg.self); Qiniu_Count_Inc(&ninterrupts); - free(task); + Qiniu_Free((void *)task); } if (ninterrupts > 0) @@ -885,22 +910,43 @@ static Qiniu_Error _Qiniu_Rio_Put( } else { - err = Qiniu_Rio_Mkfile(self, ret, key, fsize, &extra); - if (err.code == Qiniu_Rio_InvalidCtx && recorder != NULL && recorder->toLoadProgresses == Qiniu_True && fi != NULL) + const char *const *upHosts; + size_t upHostsCount; + + if (extra.upHost != NULL) { - Qiniu_Rio_PutExtra_Clear(&extra); - reinitializeRecorder(&extra, fi, recorder); - goto reinit; + upHosts = &extra.upHost; + upHostsCount = 1; } - if (err.code / 100 == 2 || err.code / 100 == 4 || err.code == Qiniu_Rio_InvalidCtx) + else { - Qiniu_Rio_clearProgresses(&extra, recorder); + upHosts = extra.upHosts; + upHostsCount = extra.upHostsCount; + } + + for (int tries = 0; tries < extra.tryTimes && tries <= self->retriesMax; tries++) + { + const char *upHost = upHosts[tries % upHostsCount]; + err = Qiniu_Rio_Mkfile(self, ret, upHost, key, fsize, &extra); + if (err.code == Qiniu_Rio_InvalidCtx && recorder != NULL && recorder->toLoadProgresses == Qiniu_True && fi != NULL) + { + Qiniu_Rio_PutExtra_Clear(&extra); + reinitializeRecorder(&extra, fi, recorder); + goto reinit; + } + else if (err.code == Qiniu_OK.code || _Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) + { + if (err.code / 100 != 4 || err.code == Qiniu_Rio_InvalidCtx) + { + Qiniu_Rio_clearProgresses(&extra, recorder); + } + break; + } } } Qiniu_Rio_PutExtra_Cleanup(&extra); - Qiniu_Free((void *)accessKey); - Qiniu_Free((void *)bucketName); + Qiniu_Multi_Free(2, (void *)accessKey, (void *)bucketName); wg.itbl->Release(wg.self); auth.itbl->Release(auth.self); self->auth = auth1; diff --git a/qiniu/resumable_io.h b/qiniu/resumable_io.h index a1efacf..f7c73c9 100644 --- a/qiniu/resumable_io.h +++ b/qiniu/resumable_io.h @@ -137,6 +137,10 @@ extern "C" // For those who want to send request to specific host. const char *upHost; + + // Specify multiple upHosts, if not set explicitly, will global QINIU_UP_HOST + const char *const *upHosts; + size_t upHostsCount; } Qiniu_Rio_PutExtra; /*============================================================================*/ @@ -157,12 +161,12 @@ extern "C" #endif QINIU_DLLAPI extern Qiniu_Error Qiniu_Rio_Put( - Qiniu_Client *self, Qiniu_Rio_PutRet *ret, - const char *uptoken, const char *key, Qiniu_ReaderAt f, Qiniu_Int64 fsize, Qiniu_Rio_PutExtra *extra); + Qiniu_Client *self, Qiniu_Rio_PutRet *ret, + const char *uptoken, const char *key, Qiniu_ReaderAt f, Qiniu_Int64 fsize, Qiniu_Rio_PutExtra *extra); QINIU_DLLAPI extern Qiniu_Error Qiniu_Rio_PutFile( - Qiniu_Client *self, Qiniu_Rio_PutRet *ret, - const char *uptoken, const char *key, const char *localFile, Qiniu_Rio_PutExtra *extra); + Qiniu_Client *self, Qiniu_Rio_PutRet *ret, + const char *uptoken, const char *key, const char *localFile, Qiniu_Rio_PutExtra *extra); /*============================================================================*/