Title: howto distcc and crossdev
Subject: howto setup emerge with distcc and crossdev
By: John Stile
Summery:  my pico-itx is wimpy but my amd64 is beefy.
          This lets the amd64 compile ebuilds for the pico.
	  I distinguish the target config from the helper config.
###########################################################
# References:
#
# http://www.gentoo.org/doc/en/distcc.xml
# http://www.gentoo.org/doc/en/cross-compiling-distcc.xml
# http://distcc.samba.org/security.html
#
###########################################################
#
# TERMS:
#  Target = host we are compiling stuff for. Often called local host.
#  Helper = host we send build jobs to. Often called remote host.
#
###########################################################
#
# 1. Collect Target Host Info For Crossdev Setup on Helper.
#
############################################################
#
# Collect Target host information in order to build 
# cross-compile tool chain on the Helper
#
#-----------------------------------------------------------
#
# CHOST
#
grep CHOST /etc/make.conf

  CHOST="i686-pc-linux-gnu"

#
# Kernel Version
#
uname -r

  2.6.29-gentoo-r3

#
# Gcc Version
#
gcc-config  -l

  [1] i686-pc-linux-gnu-4.1.2 *
  [2] i686-pc-linux-gnu-4.3.2

eix -I sys-devel/gcc |grep 4.1.2
 
       (4.1)   4.1.2!s
     Installed versions:  4.1.2(4.1)!s(04:38:31 PM 02/24/2009)(fortran gcj gtk mudflap nls -altivec -bootstrap -build -d -doc -hardened -ip28 -ip32r10k -libffi -multilib -multislot -n32 -n64 -nocxx -objc -objc++ -objc-gc -test -vanilla)

#
# Binutils Version
#
eix -I binutils

  [I] sys-devel/binutils
       Available versions:  [P]2.16.1-r3 [P]*2.16.91.0.6 2.18-r3 ~2.18-r4 ~2.19.1-r1 **2.19.51.0.13 **2.19.51.0.14 **2.20.51.0.1 {gold multislot multitarget nls test vanilla}
       Installed versions:  2.18-r3(04:59:26 09/16/09)(nls -gold -multislot -multitarget -test -vanilla)
       Homepage:	    http://sources.redhat.com/binutils/
       Description:	    Tools necessary to build programs

binutils-config --get-current-profile
  i686-pc-linux-gnu-2.18

#-----------------------------------------------------------
#
# Helper host information doesn't matter.
# We will cross-compile tool chain on the Helper for this Target
#

###########################################################
#
# 2. Install stuff on Target and Helper
#
############################################################

#-----------------------------------------------------------
#
# Target:
#--------
#
# Install distcc
#
echo 'sys-devel/distcc gtk' >> /etc/portage/package.use
emerge sys-devel/distcc  

#-----------------------------------------------------------
#
# Helper:
#--------
#
# Install distcc and crossdev
#
echo 'sys-devel/crossdev-wrappers' >> /etc/portage/package.keywords
echo 'sys-devel/distcc gtk' >> /etc/portage/package.use
emerge sys-devel/crossdev sys-devel/crossdev-wrappers sys-devel/distcc -ta


###########################################################
#
# 3. Configure stuff on Target and Helper
#
############################################################

# NOTE:  For some reason I couldn't get crossdev to build with gcc < 3.4

#-----------------------------------------------------------
#
# Target:
#--------
#
# Setting up Portage to use Distcc on the Target
#
vi /etc/make.conf

Add:
  MAKEOPTS="-j4"                            # total number of distcc processors 
  FEATURES="distcc"                         # adds distcc to emerge

#
# Specifying Participating Hosts
#
/usr/bin/distcc-config --set-hosts "192.168.0.60/4 127.0.0.1/1" 
# Or edit /etc/distcc/hosts

#
# Rules for /etc/distcc/hosts
#
# One line with all helper hosts,
# sperate hosts by a space
# Follow each host with  slash and number of processes for that host
# For a host:  <IP>/<#-of-processes>
# Example:  127.0.0.1/2    Means run 2 jobs on localhost
# Example:  192.168.0.5/8  Means run 8 jobs on 192.168.0.5
# Example:  foo.my.com/4   Means run 4 jobs on foo.my.com
# 

#
# Main config file for distcc#
# Controls what hosts can use this Target as a Helper
#
#   add --allow directive to IP of hosts who can use this machine as Helper
#   add --listen directive to tell the distcc daemon what IP to listen on
#
vi /etc/conf.d/distccd

# My Settings:
  DISTCCD_OPTS=""
  DISTCCD_EXEC="/usr/bin/distccd"
  DISTCCD_PIDFILE="/var/run/distccd/distccd.pid"
  DISTCCD_OPTS="${DISTCCD_OPTS} --port 3632"
  DISTCCD_OPTS="${DISTCCD_OPTS} --log-level critical"
  DISTCCD_OPTS="${DISTCCD_OPTS} --allow 127.0.0.1"   #< THIS must be IP not host name, or distccd won't start
  DISTCCD_OPTS="${DISTCCD_OPTS} -N 15"

#
# Starting the distcc daemon so the local host can use it.
#
/etc/init.d/distccd start

# PATH needs distcc before gcc
#
# If /etc/make.conf contains FEATURES="distcc" , this is taken care of
#
# Update PATH to include /usr/lib/distcc/bin before /usr/bin/gcc
# If using distcc, the path order should be ccache, then distcc, then gcc. 
#
export PATH="/usr/lib/ccache/bin:/usr/lib/distcc/bin:${PATH}"

#-------------------------------------------------
# NOTE: Found discussion on setting DISTCC_HOSTS: http://bugs.gentoo.org/28079
#
# From what I glean from bugzilla 28079,
# Don't use /etc/make.conf to sore DISTCC_HOSTS
# /usr/bin/distcc-config is supposed to update /etc/distcc/hosts
#
# Quote "The reason distcc-config (now) uses /etc/distcc/hosts is so that the hosts is
# set in _one_ place.  The usage of this file is also so that each user of the
# system can use it, and overwrite it with ~/.distcc/hosts or DISTCC_HOSTS
# envvar....
# DISTCC_DIR does not change where distcc client looks for /etc/distcc/hosts."
#-------------------------------------------------


#-----------------------------------------------------------
#
# Helper:
#--------
#
# Don't setup Portage to use Distcc
# leave /etc/make.conf alone.  
# Only the Target needs: FEATURES="distcc"
#

#
# Don't  setup distcc-config (/etc/distcc/hosts).
# Only the Target needs: FEATURES="distcc"
#

#
# Main config file for distcc
#
# add --allow directive to IP of hosts that can use this Helper
# add --listen directive to tell the distcc daemon what IP to listen on
#
vi /etc/conf.d/distccd

My Settings:
  DISTCCD_OPTS=""
  DISTCCD_EXEC="/usr/bin/distccd"
  DISTCCD_PIDFILE="/var/run/distccd/distccd.pid"
  DISTCCD_OPTS="${DISTCCD_OPTS} --port 3632"
  DISTCCD_OPTS="${DISTCCD_OPTS} --log-level critical"
  DISTCCD_OPTS="${DISTCCD_OPTS} --allow 192.168.1.132"
  DISTCCD_OPTS="${DISTCCD_OPTS} --allow 192.168.0.0/24"
  DISTCCD_OPTS="${DISTCCD_OPTS} --listen 192.168.0.60"
  DISTCCD_OPTS="${DISTCCD_OPTS} -N 15"

#
# Setup distcc links to proper CHOST
#
cd /usr/lib/distcc/bin/

# Remove the default links
ls -l

    total 0
    lrwxrwxrwx 1 root root 15 Sep 17 22:24 c++ -> /usr/bin/distcc
    lrwxrwxrwx 1 root root 15 Sep 17 22:24 cc -> /usr/bin/distcc
    lrwxrwxrwx 1 root root 15 Sep 17 22:24 g++ -> /usr/bin/distcc
    lrwxrwxrwx 1 root root 15 Sep 17 22:24 gcc -> /usr/bin/distcc
    lrwxrwxrwx 1 root root 15 Sep 17 22:24 x86_64-pc-linux-gnu-c++ -> /usr/bin/distcc
    lrwxrwxrwx 1 root root 15 Sep 17 22:24 x86_64-pc-linux-gnu-g++ -> /usr/bin/distcc
    lrwxrwxrwx 1 root root 15 Sep 17 22:24 x86_64-pc-linux-gnu-gcc -> /usr/bin/distcc

rm c++ g++ gcc cc
rm -rf x86_64-pc-linux-gnu-c x86_64-pc-linux-gnu-g x86_64-pc-linux-gnu-gcc

# Create wrapper script to launch compiler for proper CHOST
cat >> i686-pc-linux-gnu-wrapper <<'EOF'
#!/bin/bash
exec /usr/lib/distcc/bin/i686-pc-linux-gnu-g${0:$[-2]} "$@"
EOF
chmod a+x i686-pc-linux-gnu-wrapper

# Create target CHOST links to compiler
ln -s i686-pc-linux-gnu-wrapper cc
ln -s i686-pc-linux-gnu-wrapper gcc
ln -s i686-pc-linux-gnu-wrapper g++
ln -s i686-pc-linux-gnu-wrapper c++

# Create target CHOST links to distcc
ln -s /usr/bin/distcc i686-pc-linux-gnu-c++
ln -s /usr/bin/distcc i686-pc-linux-gnu-g++
ln -s /usr/bin/distcc i686-pc-linux-gnu-gcc

# Now looks like this
ls -l

   total 4
   lrwxrwxrwx 1 root root  25 Sep 17 22:38 c++ -> i686-pc-linux-gnu-wrapper
   lrwxrwxrwx 1 root root  25 Sep 17 22:38 cc -> i686-pc-linux-gnu-wrapper
   lrwxrwxrwx 1 root root  25 Sep 17 22:38 g++ -> i686-pc-linux-gnu-wrapper
   lrwxrwxrwx 1 root root  25 Sep 17 22:38 gcc -> i686-pc-linux-gnu-wrapper
   lrwxrwxrwx 1 root root  15 Sep 17 22:41 i686-pc-linux-gnu-c++ -> /usr/bin/distcc
   lrwxrwxrwx 1 root root  15 Sep 17 22:41 i686-pc-linux-gnu-g++ -> /usr/bin/distcc
   lrwxrwxrwx 1 root root  15 Sep 17 22:41 i686-pc-linux-gnu-gcc -> /usr/bin/distcc
   -rwxr-xr-x 1 root root 144 Sep 17 22:37 i686-pc-linux-gnu-wrapper

#
# Change Helper machine's compiler to what Target's compiler uses
# Make certain all hosts have the same GCC version. 
#
gcc-config  -l

   [1] i686-pc-linux-gnu-4.3.2 *

   [2] x86_64-pc-linux-gnu-4.1.2 *
   [3] x86_64-pc-linux-gnu-4.3.2

gcc-config  3

#
# This file should contain these lines
#
cat /etc/portage/categories

  perl-gcpancross-i686-pc-linux-gnu
  cross-i686-pc-linux-gnu

#
# Create Toolchain On Helper for the Target platform
#
crossdev -v --target i686-pc-linux-gnu --g 4.3.2-r3 --b 2.18-r3 --l 2.9_p20081201-r2 --k 2.6.29-gentoo-r3
- or - 
crossdev -v --target i686-pc-linux-gnu -S

   --------------------------------------------------------------------------------------------------------
    * Host Portage ARCH:     amd64
    * Target Portage ARCH:   x86
    * Target System:	     i686-pc-linux-gnu
    * Stage:		     4 (C/C++ compiler)

    * binutils: 	     binutils-2.18-r3
    * gcc:		     gcc-4.3.2-r3
    * headers:  	     linux-headers-2.6.29-gentoo-r3
    * libc:		     glibc-2.9_p20081201-r2

    * PORTDIR_OVERLAY:       /usr/local/overlays/mine
    * PORT_LOGDIR:	     /var/log/portage
    * PKGDIR:		     /usr/portage/packages/cross/i686-pc-linux-gnu
    * PORTAGE_TMPDIR:	     /var/tmp/cross/i686-pc-linux-gnu
     _  -  ~  -  _  -  ~  -  _  -  ~  -  _  -  ~  -  _  -  ~  -  _  -  ~  -  _  -  ~  -  _  -  ~  -  _  -
    * Forcing the latest versions of {binutils,gcc}-config/gnuconfig ...			     [ ok ]
    * Log: /var/log/portage/cross-i686-pc-linux-gnu-binutils.log
    * Emerging cross-binutils ...
   --- Invalid atom in /etc/portage/package.mask: >cross-i686-pc-linux-gnu/linux-headers-2.6.29-gentoo-r3

#
# NOTE: you might be able to get away with just
# crossdev -v --target i686-pc-linux-gnu -S
#

#
# For some reason stuff is added to /etc/portage/package.mask
#
   >cross-i686-pc-linux-gnu/binutils-2.18-r3
   >cross-i686-pc-linux-gnu/gcc-4.3.2-r3
   >cross-i686-pc-linux-gnu/linux-headers-2.6.29-gentoo-r3
   >cross-i686-pc-linux-gnu/glibc-2.9_p20081201-r2

#
# This creates the following:
#
ls -tlr /var/tmp/cross/i686-pc-linux-gnu/portage/cross-i686-pc-linux-gnu/

   total 0
   drwxrwxr-x 7 portage portage 240 Sep 18 02:37 gcc-4.1.2
   drwxrwxr-x 4 portage portage  96 Sep 18 13:01 gcc-4.2.4-r1
   drwxrwxr-x 6 portage portage 144 Sep 18 13:05 binutils-2.17-r2


#
# Starting the distcc daemon
#
/etc/init.d/distccd start



###########################################################
#
# 4. Need to clear a lock 
#
############################################################
rm -rf /var/tmp/portage/.distcc/lock/*
rm -rf /root/.distcc
rm -rf /var/tmp/portage/.distcc
rm -rf /root/.distcc
rm -rf /var/tmp/portage/.distcc


###########################################################
#
# 5. Test connection From Target To Helper 
#
############################################################
From shell on Target, telnet to port Helper listens on

  telnet <Helper IP> 3632

###########################################################
#
# 6. Monitor
#
############################################################
# Terminal-1:
# Monitor /var/tmp/portage/.distcc/* while it's running.
#
watch -n1 "ls -lR /var/tmp/portage/.distcc"

# Terminal-2:
# emerge, sending stdout to a file,
#
tail -f update_20090915.log |grep '192.168.0.60'

# Terminal-3:
# emerge, sending stdout to a file,
#
tail -f /var/log/emerge.log
#
# GUI
#
DISTCC_DIR=/var/tmp/portage/.distcc distccmon-gnome


#######################
#----------------------
#######################
#----------------------
#######################
#----------------------
#######################
#----------------------
#######################