ScrollView com um ListView não rola - android

Meu problema com o layout a seguir é que o layout não rola. Minha exigência é que eu tenha que exibir 2 listas que às vezes podem ser grandes. Então eu coloco ambos os ListViews dentro de um ScrollView para que eu possa rolar para baixo e dar uma olhada em ambos os ListViews.I don't want the ListView to scroll. Eu só quero que o meu ScrollView se comporte como deveria. Alguma sugestão??

<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="fill_parent"
android:layout_height="fill_parent"
android:orientation="vertical" >
<ScrollView
        android:id="@+id/scrollViewContactDetails"
        android:layout_width="match_parent"
        android:layout_height="fill_parent" android:fillViewport="true">


        <LinearLayout
            android:layout_width="fill_parent"
            android:layout_height="fill_parent"
            android:orientation="vertical" >


            <TextView
                android:id="@+id/name"
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                android:layout_marginLeft="10dp"
                android:layout_marginTop="5dp"
                android:textAppearance="?android:attr/textAppearanceLarge"
                android:textSize="22dp" />

            <LinearLayout
                android:id="@+id/numbersLayout"
                android:layout_width="fill_parent"
                android:layout_height="wrap_content"
                android:layout_marginLeft="20dp"
                android:layout_marginRight="20dp"
                android:orientation="vertical"
                android:weightSum="1" >

                <LinearLayout
                    android:id="@+id/numbersLayoutList"
                    android:layout_width="match_parent"
                    android:layout_height="fill_parent"
                    android:gravity="center"
                    android:orientation="vertical"
                    android:weightSum="1" >

                    <ListView
                        android:id="@+id/numbersList"
                        android:layout_width="match_parent"
                        android:layout_height="fill_parent"
                        android:layout_gravity="center"
                        android:dividerHeight="2dp" >
                    </ListView>
                </LinearLayout>
            </LinearLayout>

            <LinearLayout
                android:id="@+id/emailLayout"
                android:layout_width="fill_parent"
                android:layout_height="wrap_content"
                android:layout_marginLeft="20dp"
                android:layout_marginRight="20dp"
                android:layout_marginTop="10dp"
                android:orientation="horizontal"
                android:weightSum="1" >

                <LinearLayout
                    android:layout_width="120dp"
                    android:layout_height="match_parent"
                    android:layout_gravity="right|center"
                    android:gravity="top|right"
                    android:orientation="vertical" >

                    <TextView
                        android:id="@+id/emailId"
                        android:layout_width="120dp"
                        android:layout_height="wrap_content"
                        android:layout_marginTop="10dp"
                        android:gravity="right|center"
                        android:textAppearance="?android:attr/textAppearanceMedium"
                        android:textColor="#8d8d8d"
                        android:textSize="18dp" />
                </LinearLayout>

                <LinearLayout
                    android:id="@+id/emailLayoutList"
                    android:layout_width="match_parent"
                    android:layout_height="fill_parent"
                    android:layout_marginLeft="20dp"
                    android:gravity="center"
                    android:orientation="vertical"
                    android:weightSum="1" >

                    <ListView
                        android:id="@+id/emailList"
                        android:layout_width="match_parent"
                        android:layout_height="fill_parent"
                        android:layout_gravity="center"
                        android:layout_weight="1"
                        android:dividerHeight="2dp" >
                    </ListView>
                </LinearLayout>
            </LinearLayout>
        </LinearLayout>
    </ScrollView>
</LinearLayout>

questionAnswers(5)

yourAnswerToTheQuestion