Rethinking packaging, development and deployment

@ Pycon US 2015

Python runtime dependency graph


$ nix-store -qR `which python`
nix/store/qwwxgsg6l33lhx0v75mgmb077qggh8yl-linux-headers-3.12.32
/nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20
/nix/store/584hmj4dvlp9aj9n4kcc52a5wz1aq9ac-zlib-1.2.8
/nix/store/lg4pnma41vc1vvlb4qsriphk7sq4762r-gcc-4.8.3
/nix/store/jd3gggw5bs3a6sbjnwhjapcqr8g78f5c-attr-2.4.47
/nix/store/c2p56z920h4mxw12pjw053sqfhhh0l0y-acl-2.2.52
/nix/store/wc472nw0kyw0iwgl6352ii5czxd97js2-coreutils-8.23
/nix/store/g9ybksy400pfn7fncw8dqfnz6m7fdyrk-perl-5.20.1
/nix/store/r5sxfcwq9324xvcd1z312kb9kkddqvld-bash-4.3-p30
/nix/store/2x9j7m0amgkhjg5wc8kw5qz0nxbj5iji-openssl-1.0.1m
/nix/store/dqmh55k38i24h9xnb4qb5pqggfl4dvxm-bzip2-1.0.6
/nix/store/qbhd1fs53xwfdl379m03i3w7d84laznm-python-2.7.9

$PREFIX


$ tree /nix/store/584hmj4dvlp9aj9n4kcc52a5wz1aq9ac-zlib-1.2.8
/nix/store/584hmj4dvlp9aj9n4kcc52a5wz1aq9ac-zlib-1.2.8
├── include
│   ├── zconf.h
│   └── zlib.h
├── lib
│   ├── libz.a
│   ├── libz.so -> libz.so.1.2.8
│   ├── libz.so.1 -> libz.so.1.2.8
│   ├── libz.so.1.2.8
│   └── pkgconfig
│       └── zlib.pc
└── share
    └── man
        └── man3
            └── zlib.3.gz


$ ldd `which python`
linux-vdso.so.1 (0x00007fffc0b14000)
libgcc_s.so.1 => /nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20/lib/libgcc_s.so.1 (0x00007f829b125000)
libpython2.7.so.1.0 => /nix/store/qbhd1fs53xwfdl379m03i3w7d84laznm-python-2.7.9/lib/libpython2.7.so.1.0 (0x00007f829ad1d000)
libpthread.so.0 => /nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20/lib/libpthread.so.0 (0x00007f829ab01000)
libdl.so.2 => /nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20/lib/libdl.so.2 (0x00007f829a8fd000)
libutil.so.1 => /nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20/lib/libutil.so.1 (0x00007f829a6fa000)
libm.so.6 => /nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20/lib/libm.so.6 (0x00007f829a3f7000)
libc.so.6 => /nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20/lib/libc.so.6 (0x00007f829a05a000)
/nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20/lib/ld-linux-x86-64.so.2 (0x00007f829b33b000)

User Environments / Profiles


$ cat default.nix


derivation {
  name = "plan9port-1.0";
  builder = ./builder.sh;
  system = "x86-64-linux";
  src = /home/user/plan9port.tar.gz;
}


$ cat builder.sh


tar xvfz $src
cd plan9port/
mkdir -p $out/bin
cp planport.sh $out/bin/planport


{ stdenv, fetchurl, fetchgit, openssl, zlib, pcre, libxml2 }:

stdenv.mkDerivation rec {
  name = "nginx-${version}";
  version = "1.4.4";

  src = fetchurl {
    url = "http://nginx.org/download/nginx-${version}.tar.gz";
    sha256 = "1f82845mpgmhvm151fhn2cnqjggw9w7cvsqbva9rb320wmc9m63w";
  };

  buildInputs = [ openssl zlib pcre libxml2 ];
  configureFlags = [ "--with-http_spdy_module" ];

  meta = with stdenv.lib; {
    description = "A reverse proxy and lightweight webserver";
    maintainers = [ maintainers.domenkozar ];
    platforms = platforms.all;
    license = licenses.bsd2;
  };
}


{ fetchurl, buildPythonPackage, freetype
, zlib, libwebp, libtiff, libjpeg }:

buildPythonPackage rec {
  name = "Pillow-2.3.0";

  src = fetchurl {
    url = "http://pypi.python.org/packages/source/P/Pillow/${name}.zip";
    sha256 = "1ilf9wdp3wna5pmvxill8x08rb9gw86qkc2zwm3xk9hpy8l9pf7l";
  };

  buildInputs = [ freetype libjpeg zlib libtiff libwebp ];
 
  meta = with stdenv.lib; {
    homepage = http://python-imaging.github.com/Pillow;
    description = "Fork of The Python Imaging Library (PIL)";
    license = "http://www.pythonware.com/products/pil/license.htm";
    maintainers = [ maintainers.domenkozar ];
  };
};

Build


$ nix-build -A python
/nix/store/pbi1lgank10fy0xpjckbdpgacqw34dsz-python-2.7.9

Run


$ ls -la result
result -> /nix/store/pbi1lgank10fy0xpjckbdpgacqw34dsz-python-2.7.9

$ ./result/bin/python
Python 2.7.9 (default, Jan 01 1970, 00:00:01) 
[GCC 4.8.3] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>>

Deploy


$ nix-copy-closure --sign result/ [email protected]

Build Environment

nix-shell: virtualenv for all the software

Create user environments on the fly


$ nix-shell -p git libxml2 libxslt libjpeg libpng zlib
[nix-shell] $

Determinism / Purity


$ nix-shell -p pypyPackages.tox --pure
[nix-shell] $ less

The program ‘less’ is currently not installed. It is provided by
several packages. You can install it by typing one of the following:
  nix-env -i busybox
  nix-env -i less

Hackable / User API

Override top-level function parameters:


mySecurePython27 = pkgs.python27.override { openssl = openssl102a; }

Global overrides:


nixpkgs.config.packageOverrides = pkgs:
  { python27 = pkgs.python27.override { openssl = openssl102a; };
};

Override derivation attributes:


devPython34 = pkgs.python34.overrideDerivation (attrs:
  name = "python3.4-dev";
  src = ./.; 
})

NixOS: packages + systemd + Linux kernel


{
  boot.loader.grub.device = "/dev/sda";
  fileSystems."/".device = "/dev/sda1";

  networking.firewall = {
    enable = true;
    allowedTCPPorts = [ 8080 ];
  };

  environment.systemPackages = with pkgs; [
    wget
    git
    gnupg
    tmux
  ];
  
  systemd.services.myproject = {
    after = [ "network.target" ];
    wantedBy = [ "multi-user.target" ];
    environment.PYTHONPATH = myproject.pythonPath;
    serviceConfig = {
      ExecStart = "${pkgs.pythonPackages.pyramid}/bin/pserve ${productionini}";
      User = "myuser";
      Group = "myuser";
    };
  };
}

cat trivial.nix


{
  machine1 = { config, pkgs, ... }: {
    ..NixOS config..
  };
}

cat trivial-virtualbox.nix:


{
  machine1 = { config, pkgs, ... }: {
    deployment.targetEnv = "virtualbox";
    deployment.virtualbox.memorySize = 1024; # megabytes
  };
}

Provision and deploy


$ nixops create -d trivial ./trivial.nix ./trivial-virtualbox.nix 
33bced96-5f26-11e1-b9d7-9630d48abec1

$ nixops deploy -d trivial
creating VirtualBox VM ‘webserver’...

Hydra - build/test farm

Community size

~300 people on #nixos FreeNode channel

Multiple outputs?


$ du -sch $(nix-store -qR $(nix-build -A python))
5.1M    /nix/store/qwwxgsg6l33lhx0v75mgmb077qggh8yl-linux-headers-3.12.32
34M     /nix/store/la5imi1602jxhpds9675n2n2d0683lbq-glibc-2.20
364K    /nix/store/584hmj4dvlp9aj9n4kcc52a5wz1aq9ac-zlib-1.2.8
88M     /nix/store/lg4pnma41vc1vvlb4qsriphk7sq4762r-gcc-4.8.3
336K    /nix/store/jd3gggw5bs3a6sbjnwhjapcqr8g78f5c-attr-2.4.47
472K    /nix/store/c2p56z920h4mxw12pjw053sqfhhh0l0y-acl-2.2.52
16M     /nix/store/wc472nw0kyw0iwgl6352ii5czxd97js2-coreutils-8.23
53M     /nix/store/g9ybksy400pfn7fncw8dqfnz6m7fdyrk-perl-5.20.1
6.7M    /nix/store/r5sxfcwq9324xvcd1z312kb9kkddqvld-bash-4.3-p30
7.7M    /nix/store/2x9j7m0amgkhjg5wc8kw5qz0nxbj5iji-openssl-1.0.1m
328K    /nix/store/dqmh55k38i24h9xnb4qb5pqggfl4dvxm-bzip2-1.0.6
42M     /nix/store/qbhd1fs53xwfdl379m03i3w7d84laznm-python-2.7.9
252M    total