ожно избежать кругового перенаправления с помощью HttpClient 4.1.1. Как я получаю ошибку, как это: -

executing requestGET http://home.somehost.com/Mynet/pages/cHome.xhtml HTTP/1.1
org.apache.http.client.ClientProtocolException
    at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:822)
    at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:754)
    at edu.uci.ics.crawler4j.url.WebURL.setURL(WebURL.java:122)
    at edu.uci.ics.crawler4j.crawler.CrawlController.addSeed(CrawlController.java:207)
    at edu.uci.ics.crawler4j.example.advanced.Controller.main(Controller.java:31)
Caused by: org.apache.http.client.CircularRedirectException: Circular redirect to 'http://home.somehost.com/Mynet/pages/Home.xhtml'
    at org.apache.http.impl.client.DefaultRedirectStrategy.getLocationURI(DefaultRedirectStrategy.java:168)
    at org.apache.http.impl.client.DefaultRedirectStrategy.getRedirect(DefaultRedirectStrategy.java:193)
    at org.apache.http.impl.client.DefaultRequestDirector.handleResponse(DefaultRequestDirector.java:1021)
    at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:482)
    at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:820)

Это мой код ...

DefaultHttpClient client = null;

        try
        {
            // Set url
            //URI uri = new URI(url.toString());

            client = new DefaultHttpClient();

            client.getCredentialsProvider().setCredentials(
                    new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM),
                    new UsernamePasswordCredentials("test", "test"));


            URL url1 = new URL (url);
            HttpURLConnection connection = (HttpURLConnection) url1.openConnection();
            connection.setFollowRedirects(false);

            HttpGet request = new HttpGet(url);
            final HttpParams params = new BasicHttpParams();
            HttpClientParams.setRedirecting(params, false);
            HttpContext context = new BasicHttpContext();

            System.out.println("----------------------------------------");
            System.out.println("executing request" + request.getRequestLine());
            HttpResponse response = client.execute(request, context);
            HttpEntity entity = response.getEntity();


            System.out.println(response.getStatusLine());
                    InputStream content = entity.getContent();
                    BufferedReader in   = 
                        new BufferedReader (new InputStreamReader (content));
                    String line;
                    while ((line = in.readLine()) != null) {
                       // System.out.println(line);
                    }
                } catch(Exception e) {
                    e.printStackTrace();
                }