Matplotlib figure / plot / canvas / layout oculta botones. La imagen debería explicar el problema: me gustaría que el contenedor de la trama deje espacio para los botones de la derecha.

Aquí hay un ejemplo de código de trabajo mínimo con algunas cosas comentadas que probé:

import sys
from PyQt5.QtWidgets import QPushButton, QVBoxLayout, QApplication, QDialog
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT as NavigationToolbar
import matplotlib.pyplot as plt
import random

from PyQt5 import QtCore, QtGui, QtWidgets

class Ui_Dialog(QDialog):
    def __init__(self, parent=None):
        super(Ui_Dialog, self).__init__(parent)
        self.setupUi(self)
        self.setupPlot()

    def setupUi(self, Dialog):
        Dialog.setObjectName("Dialog")
        Dialog.resize(400, 300)
        Dialog.setToolTip("")
        Dialog.setStatusTip("")
        Dialog.setWhatsThis("")
        Dialog.setAccessibleName("")
        Dialog.setAccessibleDescription("")
        self.buttonBox = QtWidgets.QDialogButtonBox(Dialog)
        self.buttonBox.setGeometry(QtCore.QRect(320, 50, 81, 241))
        self.buttonBox.setOrientation(QtCore.Qt.Vertical)
        self.buttonBox.setStandardButtons(QtWidgets.QDialogButtonBox.Cancel|QtWidgets.QDialogButtonBox.Ok)
        self.buttonBox.setObjectName("buttonBox")
        self.retranslateUi(Dialog)
        self.buttonBox.accepted.connect(Dialog.accept)
        self.buttonBox.rejected.connect(Dialog.reject)
        QtCore.QMetaObject.connectSlotsByName(Dialog)

    def retranslateUi(self, Dialog):
        pass

    def setupPlot(self):
        self.figure = plt.figure()
        self.canvas = FigureCanvas(self.figure)
        #self.canvas.resize(self, 5, 5)
        #self.resize(5,5)

        self.toolbar = NavigationToolbar(self.canvas, self)
        layout = QVBoxLayout()
        layout.addWidget(self.toolbar)
        layout.addWidget(self.canvas)
        self.setLayout(layout)
        self.plot()

    def plot(self):
        data = [random.random() for i in range(10)]
        self.figure.clear()

        #plt.figure(num=2, figsize=(3, 3), dpi=80, facecolor='w', edgecolor='k')
        #self.figure.set_size_inches(5, 5, forward=True)

        ax = self.figure.add_subplot(111)

        #self.figure.subplots_adjust(left=1, bottom=1, right=1, top=1, wspace=1, hspace=1)

        ax.plot(data, '*-')
        self.canvas.draw()


if __name__ == '__main__':
    app = QApplication(sys.argv)

    main = Ui_Dialog()
    main.show()

    sys.exit(app.exec_())