JFrame com vários JPanel

Este não é o código do meu programa (pois ele é bastante grande) mas é um programa parecido, mais simples, e com o mesmo problema.

A variavel int x é só para ajudar a perceber quantas vezes ele corre o actionlistener até encontrar o botao accionado. O system.out.println() foi para conseguir ver.

[i]

import java.lang.<em>;

import java.awt.</em>;

import java.awt.event.<em>;

import java.util.</em>;

import javax.swing.*;

static JFrame frame;

int x=0;

JButton button1 = new JButton("1");
JButton button2 = new JButton("2");

JLabel labelTeste = new JLabel("Teste", JLabel.CENTER);

public Main(){
	JPanel painel = new JPanel(new GridBagLayout());
	GridBagConstraints c = new GridBagConstraints();
	
	c.gridx = 0;
	c.gridy=1;
	button1.addActionListener(this);
	painel.add(button1,c);
	
	c.gridx=0;
	c.gridy=0;
	c.gridwidth=2;
	painel.add(labelTeste,c);
	
	frame.getContentPane().add(painel, BorderLayout.CENTER);
	frame.pack();
	frame.setVisible(true);
}

public void botao1(){
	frame.getContentPane().removeAll();
	
	JPanel painel = new JPanel(new GridBagLayout());
	GridBagConstraints c = new GridBagConstraints();
	
	c.gridx = 0;
	c.gridy=1;
	button1.addActionListener(this);
	painel.add(button1,c);
	
	c.gridx=0;
	c.gridy=0;
	c.gridwidth=2;
	painel.add(labelTeste,c);
	
	frame.getContentPane().add(painel, BorderLayout.CENTER);
	frame.pack();
	frame.setVisible(true);
}

public void botao2(){
	frame.getContentPane().removeAll();
	
	JPanel painel = new JPanel(new GridBagLayout());
	GridBagConstraints c = new GridBagConstraints();
	
	c.gridx = 0;
	c.gridy=1;
	button2.addActionListener(this);
	painel.add(button2,c);
	
	c.gridx=0;
	c.gridy=0;
	c.gridwidth=2;
	painel.add(labelTeste,c);
	
	frame.getContentPane().add(painel, BorderLayout.CENTER);
	frame.pack();
	frame.setVisible(true);
}

public void actionPerformed(ActionEvent event)
{
	
	Object src = event.getSource();
	if(src==button1) botao2();
	if(src==button2) botao1();
	System.out.println("actionlistener "+x);
	x++;
}

public static void start(){
	JFrame.setDefaultLookAndFeelDecorated(true);		
	Main start2 = new Main();
}

public static void main(String[] args) 
{
	frame = new JFrame("Hanovia");
	frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
	
	javax.swing.SwingUtilities.invokeLater(new Runnable() {
		public void run(){
			start();
		}
	});
}

import java.awt.GridBagConstraints;
import java.awt.GridBagLayout;
import java.awt.Insets;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;

import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
import javax.swing.SwingUtilities;
import javax.swing.WindowConstants;

public class ExemploAction extends JFrame implements ActionListener {

	private static final long serialVersionUID = 1L;
	private JButton botao1;
	private JButton botao2;
	private JTextArea editorStatus;
	private JScrollPane scrollPane;

	public ExemploAction() {
		super();
		initGUI();
	}

	public static void main(final String[] args) {
		SwingUtilities.invokeLater(new Runnable() {
			@Override
			public void run() {
				final ExemploAction inst = new ExemploAction();
				inst.setLocationRelativeTo(null);
				inst.setVisible(true);
			}
		});
	}

	@Override
	public void actionPerformed(final ActionEvent arg0) {
		getEditorStatus().append(
				((JButton) arg0.getSource()).getText() + " foi pressionado!\n");
	}

	private JButton getBotao1() {
		if (botao1 == null) {
			botao1 = new JButton();
			botao1.setText("Botão 1");
			botao1.addActionListener(this);
		}
		return botao1;
	}

	private JButton getBotao2() {
		if (botao2 == null) {
			botao2 = new JButton();
			botao2.setText("Botão 2");
			botao2.addActionListener(this);
		}
		return botao2;
	}

	private JTextArea getEditorStatus() {
		if (editorStatus == null) {
			editorStatus = new JTextArea();
			editorStatus.setAutoscrolls(true);
			editorStatus.setEditable(false);
			editorStatus.setEnabled(false);

		}
		return editorStatus;
	}

	private JScrollPane getScrollPane() {
		if (scrollPane == null) {
			scrollPane = new JScrollPane();
			scrollPane.setViewportView(getEditorStatus());
		}
		return scrollPane;
	}

	private void initGUI() {
		setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE);

		setLayout(new GridBagLayout());
		final GridBagConstraints constraints = new GridBagConstraints(0, 0, 1,
				1, 0.1, 0.1, GridBagConstraints.PAGE_START,
				GridBagConstraints.NONE, new Insets(10, 3, 3, 3), 0, 0);
		
		add(getBotao1(), constraints);
		constraints.gridx = 1;
		add(getBotao2(), constraints);
		constraints.gridx = 0;
		constraints.gridy = 1;
		constraints.gridwidth = 2;
		constraints.weighty = 1.0;
		constraints.fill = GridBagConstraints.BOTH;
		constraints.anchor = GridBagConstraints.CENTER;
		add(getScrollPane(), constraints);
		
		setSize(400, 300);
		getEditorStatus().append("Programa iniciado!\n");
	}

}