added EC2 provisioning, optimised docker for EC2, added python and

cppjit client (python not yet working)

8 files changed, 140 insertions, 28 deletions

diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml
new file mode 100644
index 000000000..767fc7ba6
--- /dev/null
+++ b/ansible/roles/common/handlers/main.yml
@@ -0,0 +1,4 @@
+---
+- name: restart sshd
+  sudo: true 
+  service: name=ssh state=restarted
diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml
index 3c483ce13..6c0c7a119 100644
--- a/ansible/roles/common/tasks/main.yml
+++ b/ansible/roles/common/tasks/main.yml
@@ -1,28 +1,13 @@
 ---
-- name: install docker
-  sudo: true
-  # install script from https://docs.docker.com/installation/ubuntulinux/
-  shell: curl -sSL https://get.docker.com/ubuntu/ | sudo sh  
-
 - name: install package dependencies
   sudo: true
   apt: name={{ item }}
   with_items:
-      - python-pip
-      - htop
-
-- name: install python dependencies
-  sudo: true
-  pip: name=docker-py
-
-
-- name: enable docker for standard user
-  sudo: true
-  # todo: how to logout after this command, otherwise won't be effective in this play
-  user: name=vagrant groups=docker append=yes
+    - parallel
+    - htop
 
 - name: checkout test repo
   git:
-      repo: https://github.com/sveneh/tests.git 
-      version: develop
-      dest: git
+    repo: https://github.com/ethereum/tests.git 
+    version: develop
+    dest: git
diff --git a/ansible/roles/docker/handlers/main.yml b/ansible/roles/docker/handlers/main.yml
new file mode 100644
index 000000000..767fc7ba6
--- /dev/null
+++ b/ansible/roles/docker/handlers/main.yml
@@ -0,0 +1,4 @@
+---
+- name: restart sshd
+  sudo: true 
+  service: name=ssh state=restarted
diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml
new file mode 100644
index 000000000..c434b34a5
--- /dev/null
+++ b/ansible/roles/docker/tasks/main.yml
@@ -0,0 +1,40 @@
+---
+- name: update package list
+  sudo: true
+  apt: update_cache=true
+
+- name: install docker dependencies
+  sudo: true
+  apt: name={{ item }} install_recommends=false
+  with_items:
+    # Docker has serious problems on EC2: http://www.danstutzman.com/2014/07/speed-up-docker-on-ec2
+    # and https://github.com/docker/docker/issues/4036
+    - linux-generic
+    - python-pip
+
+- name: Kernel update needs a restart 
+  sudo: true
+  command: shutdown -r now 
+  async: 0
+  poll: 0
+  ignore_errors: true
+
+- name: waiting for server to come back
+  local_action: wait_for host={{ inventory_hostname }} port=22
+                state=started
+  sudo: false
+
+- name: install docker
+  sudo: true
+  # install script from https://docs.docker.com/installation/ubuntulinux/
+  # TODO this is not idempotent
+  shell: curl -sSL https://get.docker.com/ubuntu/ | sudo sh  
+
+- name: install docker python API
+  sudo: true
+  pip: name=docker-py
+
+- name: enable docker for standard user
+  sudo: true
+  user: name={{ ansible_ssh_user }} groups=docker append=yes
+  notify: restart sshd
diff --git a/ansible/roles/ec2/tasks/setup.yml b/ansible/roles/ec2/tasks/setup.yml
new file mode 100644
index 000000000..0876d8d2d
--- /dev/null
+++ b/ansible/roles/ec2/tasks/setup.yml
@@ -0,0 +1,33 @@
+---
+- name: create default security group 
+  ec2_group:
+    name: "{{ security_group }}"
+    region: "{{ region }}"
+    description: "{{ project_description }}"
+    rules:
+        # ssh
+      - proto: tcp
+        from_port: 22
+        to_port: 22
+        cidr_ip: "{{ ip_access_range }}"
+    rules_egress:
+      - proto: all
+        cidr_ip: "{{ ip_access_range }}"
+
+
+- name: start ec2 instances
+  ec2:  
+    group: "{{ security_group }}"
+    instance_type: "{{ instance_type }}" 
+    image: "{{ image }}" 
+    wait: true 
+    region: "{{ region }}"
+    key_name: "{{ keypair }}"
+    instance_tags:
+      Name: test_runner
+    count_tag: 
+      Name: test_runner
+    exact_count: "{{ total_no_instances }}"
+#    volumes:
+#      - device_name: /dev/xvda
+#        volume_size: "{{ volume_size_gb }}"
diff --git a/ansible/roles/ec2/tasks/terminate.yml b/ansible/roles/ec2/tasks/terminate.yml
new file mode 100644
index 000000000..e3d3df672
--- /dev/null
+++ b/ansible/roles/ec2/tasks/terminate.yml
@@ -0,0 +1,8 @@
+---
+- name: Terminate ec2 instances 
+  local_action: ec2
+        state=absent
+        instance_ids={{ ec2_id }}
+        region={{ region }}
+        wait=true
+
diff --git a/ansible/roles/ec2/vars/main.yml b/ansible/roles/ec2/vars/main.yml
new file mode 100644
index 000000000..ea260c927
--- /dev/null
+++ b/ansible/roles/ec2/vars/main.yml
@@ -0,0 +1,21 @@
+---
+# default config for ec2 instances
+
+instance_type:      c4.xlarge  
+security_group:     client-tests
+
+# image:              ami-d6e7c084
+image:              ami-9eaa1cf6
+# region:             ap-southeast-1
+region:             us-east-1
+keypair:            christoph
+# keypair:            client-tests
+volume_size_gb:     50
+
+# limit access to AWS to these clients in CDIR notation
+ip_access_range:    0.0.0.0/0
+
+
+project_description: https://github.com/ethereum/tests
+
+total_no_instances: 1
diff --git a/ansible/roles/testrunner/tasks/main.yml b/ansible/roles/testrunner/tasks/main.yml
index ce7fd8efa..7998dee72 100644
--- a/ansible/roles/testrunner/tasks/main.yml
+++ b/ansible/roles/testrunner/tasks/main.yml
@@ -1,18 +1,35 @@
 ---
 - name: update C++ client
-  sudo: true  
   docker_image:
-      path: git/ansible/test-files/docker-cpp
-      name: cpp
+      path: /home/{{ ansible_ssh_user }}/git/ansible/test-files/docker-cppjit
+      name: cppjit
       state: build
+#  command: docker build -t cpp /home/{{ ansible_ssh_user }}/git/ansible/test-files/docker-cpp
+  async: 3600
+  poll: 5
 
 - name: update Go client
-  sudo: true 
   docker_image:
-      path: git/ansible/test-files/docker-go
+      path: /home/{{ ansible_ssh_user }}/git/ansible/test-files/docker-go
       name: go
       state: build
+#  command: docker build -t go /home/{{ ansible_ssh_user }}/git/ansible/test-files/docker-go 
+  async: 3600
+  poll: 5
 
-- name: Run infinite tests (press ^C to stop)
-  sudo: true
-  shell: git/ansible/test-files/testrunner.sh
+- name: update Python client
+  docker_image:
+      path: /home/{{ ansible_ssh_user }}/git/ansible/test-files/docker-python
+      name: python
+      state: build
+  async: 3600
+  poll: 5
+
+- name: Run infinite tests 
+  shell: seq {{ ansible_processor_vcpus }} | parallel --max-args=0 /home/{{ ansible_ssh_user }}/git/ansible/test-files/testrunner.sh
+  async: "{{ 3600 * 2 }}"
+  poll: 0
+  register: log_runner
+
+- name: verify previous task
+  async_status: jid={{ log_runner.ansible_job_id }}