There were some homebrew packages installed as dependencies, which I didn't note at the time, but they should be pretty apparent as the build fails. The configure flags below is the part that took a while to get right. Switching between versions is as easy as doing a "make install", and reloading apache.
php5.5
./configure --prefix=/usr/local/php5.5 -with-apxs2=/usr/sbin/apxs --disable-cgi --with-mysql=mysqlnd --with-mysqli=mysqlnd --with-pdo-mysql=mysqlnd --with-mysql-sock=/tmp/mysql.sock --with-ldap --with-gd --with-curl --enable-mbstring --with-mhash --with-mcrypt --with-openssl --enable-opcache
php5.4
./configure --prefix=/usr/local/php5.4 -with-apxs2=/usr/sbin/apxs --disable-cgi --with-mysql=mysqlnd --with-mysqli=mysqlnd --with-pdo-mysql=mysqlnd --with-mysql-sock=/tmp/mysql.sock --with-ldap --with-gd --with-curl --enable-mbstring --with-mhash --with-mcrypt --with-openssl
php5.3
./configure --prefix=/usr/local/php5.3 -with-apxs2=/usr/sbin/apxs --disable-cgi --with-mysql=mysqlnd --with-mysqli=mysqlnd --with-pdo-mysql=mysqlnd --with-mysql-sock=/tmp/mysql.sock --with-ldap --with-gd --with-curl --enable-mbstring --with-mhash --with-mcrypt --with-openssl