Pracuję z procesem, który używa interfejsu API Drive do przesyłania zwykłych plików tekstowych na Dysk Google. Proces często uderza w wyjątki dotyczące limitów szybkości, nawet jeśli rzeczywista liczba żądań nie jest w pobliżu limitu użytkownika dla interfejsu API Drive ustawionego w konsoli API. W rzeczywistości ustawienie limitu na użytkownika nie wpływa na szybkość, z jaką otrzymujemy wyjątki. Czy jest jakiś inny limit (inny niż limit na użytkownika), który reguluje liczbę żądań, które można wykonać na sekundę? Czy można to zmienić?

Proces wykorzystuje wykładnicze wyłączenie tych wyjątków, więc działania są ostatecznie skuteczne. Wykonujemy tylko około 5 żądań na sekundę, a limit na użytkownika wynosi 100.

Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden
{
  "code" : 403,
  "errors" : [ {
    "domain" : "usageLimits",
    "message" : "Rate Limit Exceeded",
    "reason" : "rateLimitExceeded"
  } ],
  "message" : "Rate Limit Exceeded"
}

EDYCJA: Oto „uproszczona” wersja kodu od programisty. Korzystamy z konta usługi z delegacją domeny zgodnie z opisem w:https://developers.google.com/drive/delegation.

package com.seto.fs.daemon;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.FileContent;
import com.google.api.client.http.HttpBackOffIOExceptionHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler.BackOffRequired;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.testing.util.MockBackOff;
import com.google.api.client.util.DateTime;
import com.google.api.client.util.ExponentialBackOff;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.Drive.Files.Insert;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.drive.model.ChildList;
import com.google.api.services.drive.model.ChildReference;
import com.google.api.services.drive.model.File.Labels;
import com.google.api.services.drive.model.ParentReference;


public class Test {
    private static final int testFilesCount = 100;
    private static final int threadsCount = 3;
    private static final AtomicInteger rateLimitErrorsCount = new AtomicInteger(0);

    private static final String impersonatedUser = "<impersonatedUserEmail>";
    private static final String serviceAccountID = "<some-id>@developer.gserviceaccount.com";
    private static final String serviceAccountPK = "/path/to/<public_key_fingerprint>-privatekey.p12";

    public static void main(String[] args) throws Exception {
        // Create HTTP transport
        HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
        // Create JsonFactory
        final JsonFactory jsonFactory = new JacksonFactory();

        // Create Google credential for service account
        final Credential credential = new GoogleCredential.Builder()
                .setTransport(httpTransport)
                .setJsonFactory(jsonFactory)
                .setServiceAccountScopes(Arrays.asList(DriveScopes.DRIVE))
                .setServiceAccountUser(impersonatedUser)
                .setServiceAccountId(serviceAccountID)
                .setServiceAccountPrivateKeyFromP12File(new File(serviceAccountPK))
                .build();

        // Create Drive client
        final Drive drive = new Drive.Builder(httpTransport, jsonFactory, new HttpRequestInitializer() {
            public void initialize(HttpRequest request) throws IOException {
                request.setContentLoggingLimit(0);
                request.setCurlLoggingEnabled(false);

                // Authorization initialization
                credential.initialize(request);

                // Exponential Back-off for 5xx response and 403 rate limit exceeded error
                HttpBackOffUnsuccessfulResponseHandler serverErrorHandler
                    = new HttpBackOffUnsuccessfulResponseHandler(new ExponentialBackOff.Builder().build());
                serverErrorHandler.setBackOffRequired(new BackOffRequired() {
                    public boolean isRequired(HttpResponse response) {
                        return response.getStatusCode() / 100 == 5
                            || (response.getStatusCode() == 403 && isRateLimitExceeded(
                                    GoogleJsonResponseException.from(jsonFactory, response)));
                    }
                });
                request.setUnsuccessfulResponseHandler(serverErrorHandler);

                // Back-off for socket connection error
                MockBackOff backOff = new MockBackOff();
                backOff.setBackOffMillis(2000);
                backOff.setMaxTries(5);
                request.setIOExceptionHandler(new HttpBackOffIOExceptionHandler(backOff));
            }
        }).setApplicationName("GoogleDriveUploadFile/1.0").build();

        // Get root folder id
        final String rootFolderId = drive.about().get().execute().getRootFolderId();

        // Query all children under root folder
        ChildList result = drive.children().list(rootFolderId).execute();

        // Delete all children under root folder
        for (ChildReference child : result.getItems()) {
            System.out.println("Delete child: " + child.getId());
            drive.files().delete(child.getId()).execute();
        }

        // Create a drive folder
        com.google.api.services.drive.model.File folderMetadata
            = new com.google.api.services.drive.model.File();
        folderMetadata.setMimeType("application/vnd.google-apps.folder")
            .setParents(Arrays.asList(new ParentReference().setId(rootFolderId)))
            .setTitle("DriveTestFolder");
        final com.google.api.services.drive.model.File driveTestFolder = drive.files().insert(folderMetadata).execute();

        // Create test files
        final List<File> testFiles = Collections.synchronizedList(createTestFiles());

        // Run threads to upload files to drive
        List<Thread> threads = new ArrayList<Thread>();
        for (int i = 0; i < threadsCount; i++) {
            Thread thread = new Thread(new Runnable() {
                public void run() {
                    while (testFiles.size() > 0) {
                        try {
                            File testFile = testFiles.remove(0);

                            // The file meta data
                            com.google.api.services.drive.model.File fileMetadata =
                                new com.google.api.services.drive.model.File()
                                .setTitle(testFile.getName()).setParents(Arrays.asList(new ParentReference().setId(driveTestFolder.getId())))
                                .setLabels(new Labels().setRestricted(false)).setMimeType("text/plain")
                                .setModifiedDate(new DateTime(testFile.lastModified()))
                                .setDescription("folder:MyDrive " + testFile.getName());

                            // Insert to drive
                            FileContent fileContent = new FileContent("text/plain", testFile);
                            Insert insertFileCommand = drive.files().insert(fileMetadata, fileContent)
                                    .setUseContentAsIndexableText(true);
                            insertFileCommand.getMediaHttpUploader().setDirectUploadEnabled(true);

                            insertFileCommand.execute();

                            System.out.println(testFile.getName() + " is uploaded");
                        } catch (IOException e) {
                            e.printStackTrace();
                        } catch (IndexOutOfBoundsException e) {
                            // ignore
                        }
                    }
                }
            });
            threads.add(thread);
        }

        long startTime = System.currentTimeMillis();

        for (Thread thread : threads) {
            thread.start();
        }

        for (Thread thread : threads) {
            thread.join();
        }

        System.out.println("Total time spent: " + (System.currentTimeMillis() - startTime)
                + "ms for " + testFilesCount + " files with " + threadsCount + " threads");
        System.out.println("Rate limit errors hit: " + rateLimitErrorsCount.intValue());
    }

    private static List<File> createTestFiles() throws Exception {

        // Create test files directory
        File testFolder = new File("TestFiles");
        testFolder.mkdir();

        // Create test files
        List<File> testFiles = new ArrayList<File>();
        for (int i = 0; i < testFilesCount; i++) {
            File testFile = new File("TestFiles/" + i + ".txt");
            FileOutputStream fops = new FileOutputStream(testFile);
            fops.write(testFile.getAbsolutePath().getBytes());
            fops.close();
            testFiles.add(testFile);
        }
        return testFiles;
    }

    private static boolean isRateLimitExceeded(GoogleJsonResponseException ex) {
        boolean result = false;
        if (ex.getDetails() != null && ex.getDetails().getErrors() != null
                && ex.getDetails().getErrors().size() > 0) {
            String reason = ex.getDetails().getErrors().get(0).getReason();
            result = "rateLimitExceeded".equals(reason) || "userRateLimitExceeded".equals(reason);
            if (result) {
                rateLimitErrorsCount.incrementAndGet();
                System.err.println("Rate limit error");
            }
        }

        return result;
    }

}

EDYCJA: Wybraliśmy ten wyjątek, gdy używamy pojedynczego wątku i wprowadzamy opóźnienie 500 milisekund między każdym wywołaniem. Wygląda na to, że nie da się uzyskać w pobliżu skonfigurowanej przez nas stawki za użytkownika. Nawet domyślne 10 żądań na sekundę wydaje się niemożliwe. Czemu?