Eu estou usando codeigniter

Esse é o meu modelo

<?php

class groups_Model extends CI_Model {
    public function __construct(){
        parent::__construct();
    }
    /**
     * get All groups
     * @return array object list of groups
     */
    public function get(){
        $this->db->select('groups.id,groups.name as group,status.name as status');
        $this->db->from('groups');
        $this->db->join('status', 'status.id = groups.status');
        $this->db->order_by('groups.id', 'ASC');
        $this->db->limit($this->config->item('per_page'),$this->uri->segment(4));

        $query = $this->db->get();
        $total = $query->num_rows();
        if($total > 0){
            return $query->result();
        }
        return false;
    }

?>  

Agora eu tenho 500000 registros fictícios no meu banco de dados
Estou buscando 20 registros usando paginação
mas minha consulta leva de 5 a 6 segundos como acelerar o desempenho do banco de dados

código do controlador parecido com este

<?php

public function groups(){
    $this->output->enable_profiler(TRUE);
    $this->benchmark->mark('code_start');
    $this->output->cache(5);
    $this->load->model('groups_Model');
    $this->load->library('pagination');
    $this->config->load('pagination');
    $config['base_url'] = base_url().'admin/auth/groups';
    $config['total_rows'] =$this->groups_Model->count(null);
    $this->pagination->initialize($config);
    $data['groupsList']=$this->groups_Model->get();
    $this->load->view('admin/templates/header');
    $this->load->view('admin/groups',$data);
    $this->load->view('admin/templates/footer');
    $this->benchmark->mark('code_end');
    echo $this->benchmark->elapsed_time('code_start', 'code_end');
}
?>  

Todas as consultas que demoram tanto tempo para serem executadas

  DATABASE:  database_name (Auth:$db)   QUERIES: 5 (5.8955 seconds)  (Hide)
1.6279      SELECT * FROM groups 
4.2598      SELECT `groups`.`id`, `groups`.`name` as `group`, `status`.`name` as `status`
FROM `groups`
JOIN `status` ON `status`.`id` = `groups`.`status`
ORDER BY `groups`.`id` ASC
 LIMIT 40, 20 
0.0078      SELECT `username`
FROM `users`
WHERE `id` = '1' 
0.0000      SELECT `first_name`
FROM `users`
WHERE `id` = '1' 
0.0000      SELECT `last_name`
FROM `users`
WHERE `id` = '1'