Aumentar o limite por usuário da API do Google Drive não impede as exceções de limite de taxa
Estou trabalhando com um processo que usa a API do Google Drive para enviar arquivos de texto simples para o Google Drive. O processo freqüentemente atinge exceções de limite de taxa, embora o número real de solicitações não esteja nem perto do limite por usuário para a API do Google Drive definida no console de APIs. Na verdade, definir o limite por usuário não parece afetar a taxa na qual recebemos exceções. Existe algum outro limite (além do limite por usuário) que está governando quantas solicitações podem ser feitas por segundo? Pode ser ajustado?
O processo usa back-off exponencial nessas exceções, de forma que as ações sejam bem-sucedidas. Estamos fazendo apenas cerca de 5 solicitações por segundo e o limite por usuário é definido como 100.
Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden
{
"code" : 403,
"errors" : [ {
"domain" : "usageLimits",
"message" : "Rate Limit Exceeded",
"reason" : "rateLimitExceeded"
} ],
"message" : "Rate Limit Exceeded"
}
EDIT: aqui está uma versão "simplificada" do código do desenvolvedor. Estamos usando a conta de serviço com delegação de domínio, conforme descrito em:https://developers.google.com/drive/delegation.
package com.seto.fs.daemon;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.FileContent;
import com.google.api.client.http.HttpBackOffIOExceptionHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler.BackOffRequired;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.testing.util.MockBackOff;
import com.google.api.client.util.DateTime;
import com.google.api.client.util.ExponentialBackOff;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.Drive.Files.Insert;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.drive.model.ChildList;
import com.google.api.services.drive.model.ChildReference;
import com.google.api.services.drive.model.File.Labels;
import com.google.api.services.drive.model.ParentReference;
public class Test {
private static final int testFilesCount = 100;
private static final int threadsCount = 3;
private static final AtomicInteger rateLimitErrorsCount = new AtomicInteger(0);
private static final String impersonatedUser = "<impersonatedUserEmail>";
private static final String serviceAccountID = "<some-id>@developer.gserviceaccount.com";
private static final String serviceAccountPK = "/path/to/<public_key_fingerprint>-privatekey.p12";
public static void main(String[] args) throws Exception {
// Create HTTP transport
HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
// Create JsonFactory
final JsonFactory jsonFactory = new JacksonFactory();
// Create Google credential for service account
final Credential credential = new GoogleCredential.Builder()
.setTransport(httpTransport)
.setJsonFactory(jsonFactory)
.setServiceAccountScopes(Arrays.asList(DriveScopes.DRIVE))
.setServiceAccountUser(impersonatedUser)
.setServiceAccountId(serviceAccountID)
.setServiceAccountPrivateKeyFromP12File(new File(serviceAccountPK))
.build();
// Create Drive client
final Drive drive = new Drive.Builder(httpTransport, jsonFactory, new HttpRequestInitializer() {
public void initialize(HttpRequest request) throws IOException {
request.setContentLoggingLimit(0);
request.setCurlLoggingEnabled(false);
// Authorization initialization
credential.initialize(request);
// Exponential Back-off for 5xx response and 403 rate limit exceeded error
HttpBackOffUnsuccessfulResponseHandler serverErrorHandler
= new HttpBackOffUnsuccessfulResponseHandler(new ExponentialBackOff.Builder().build());
serverErrorHandler.setBackOffRequired(new BackOffRequired() {
public boolean isRequired(HttpResponse response) {
return response.getStatusCode() / 100 == 5
|| (response.getStatusCode() == 403 && isRateLimitExceeded(
GoogleJsonResponseException.from(jsonFactory, response)));
}
});
request.setUnsuccessfulResponseHandler(serverErrorHandler);
// Back-off for socket connection error
MockBackOff backOff = new MockBackOff();
backOff.setBackOffMillis(2000);
backOff.setMaxTries(5);
request.setIOExceptionHandler(new HttpBackOffIOExceptionHandler(backOff));
}
}).setApplicationName("GoogleDriveUploadFile/1.0").build();
// Get root folder id
final String rootFolderId = drive.about().get().execute().getRootFolderId();
// Query all children under root folder
ChildList result = drive.children().list(rootFolderId).execute();
// Delete all children under root folder
for (ChildReference child : result.getItems()) {
System.out.println("Delete child: " + child.getId());
drive.files().delete(child.getId()).execute();
}
// Create a drive folder
com.google.api.services.drive.model.File folderMetadata
= new com.google.api.services.drive.model.File();
folderMetadata.setMimeType("application/vnd.google-apps.folder")
.setParents(Arrays.asList(new ParentReference().setId(rootFolderId)))
.setTitle("DriveTestFolder");
final com.google.api.services.drive.model.File driveTestFolder = drive.files().insert(folderMetadata).execute();
// Create test files
final List<File> testFiles = Collections.synchronizedList(createTestFiles());
// Run threads to upload files to drive
List<Thread> threads = new ArrayList<Thread>();
for (int i = 0; i < threadsCount; i++) {
Thread thread = new Thread(new Runnable() {
public void run() {
while (testFiles.size() > 0) {
try {
File testFile = testFiles.remove(0);
// The file meta data
com.google.api.services.drive.model.File fileMetadata =
new com.google.api.services.drive.model.File()
.setTitle(testFile.getName()).setParents(Arrays.asList(new ParentReference().setId(driveTestFolder.getId())))
.setLabels(new Labels().setRestricted(false)).setMimeType("text/plain")
.setModifiedDate(new DateTime(testFile.lastModified()))
.setDescription("folder:MyDrive " + testFile.getName());
// Insert to drive
FileContent fileContent = new FileContent("text/plain", testFile);
Insert insertFileCommand = drive.files().insert(fileMetadata, fileContent)
.setUseContentAsIndexableText(true);
insertFileCommand.getMediaHttpUploader().setDirectUploadEnabled(true);
insertFileCommand.execute();
System.out.println(testFile.getName() + " is uploaded");
} catch (IOException e) {
e.printStackTrace();
} catch (IndexOutOfBoundsException e) {
// ignore
}
}
}
});
threads.add(thread);
}
long startTime = System.currentTimeMillis();
for (Thread thread : threads) {
thread.start();
}
for (Thread thread : threads) {
thread.join();
}
System.out.println("Total time spent: " + (System.currentTimeMillis() - startTime)
+ "ms for " + testFilesCount + " files with " + threadsCount + " threads");
System.out.println("Rate limit errors hit: " + rateLimitErrorsCount.intValue());
}
private static List<File> createTestFiles() throws Exception {
// Create test files directory
File testFolder = new File("TestFiles");
testFolder.mkdir();
// Create test files
List<File> testFiles = new ArrayList<File>();
for (int i = 0; i < testFilesCount; i++) {
File testFile = new File("TestFiles/" + i + ".txt");
FileOutputStream fops = new FileOutputStream(testFile);
fops.write(testFile.getAbsolutePath().getBytes());
fops.close();
testFiles.add(testFile);
}
return testFiles;
}
private static boolean isRateLimitExceeded(GoogleJsonResponseException ex) {
boolean result = false;
if (ex.getDetails() != null && ex.getDetails().getErrors() != null
&& ex.getDetails().getErrors().size() > 0) {
String reason = ex.getDetails().getErrors().get(0).getReason();
result = "rateLimitExceeded".equals(reason) || "userRateLimitExceeded".equals(reason);
if (result) {
rateLimitErrorsCount.incrementAndGet();
System.err.println("Rate limit error");
}
}
return result;
}
}
EDIT: Nós atingimos essa exceção quando usamos um único thread e colocamos um atraso de 500 milissegundos entre cada chamada. Parece impossível chegar perto da taxa por usuário que configuramos. Mesmo o padrão de 10 solicitações por segundo parece ser impossível. Por quê?