Merge pull request #1506 from DataDog/quentin/clean_before_cache

[tests] Clean before cache
DataDog · Apr 7, 2015 · f4eb519 · f4eb519
2 parents f79d93d + ea4bf2e
commit f4eb519
Show file tree

Hide file tree

Showing 27 changed files with 74 additions and 45 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -12,8 +12,7 @@ python:
 
 cache:
   directories:
-    - $HOME/virtualenv/python2.6.9/lib/python2.6/site-packages
-    - $HOME/virtualenv/python2.7.9/lib/python2.7/site-packages
+    - $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION.9
 
 matrix:
   fast_finish: true

diff --git a/ci/apache.rb b/ci/apache.rb
@@ -66,7 +66,10 @@ def apache_rootdir
       Rake::Task['ci:common:run_tests'].invoke(this_provides)
     end
 
-    task :before_cache => ['ci:common:before_cache']
+    task :before_cache => ['ci:common:before_cache'] do
+      # Useless to cache the conf, as it is regenerated every time
+      sh %(rm -f #{apache_rootdir}/conf/httpd.conf)
+    end
 
     task :cache => ['ci:common:cache']
 
@@ -90,7 +93,7 @@ def apache_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/cassandra.rb b/ci/cassandra.rb
@@ -42,12 +42,14 @@ def cass_rootdir
       Rake::Task['ci:common:run_tests'].invoke(this_provides)
     end
 
-    task :before_cache => ['ci:common:before_cache']
+    task :before_cache => :cleanup
 
     task :cache => ['ci:common:cache']
 
     task cleanup: ['ci:common:cleanup'] do
       sh %(kill `cat $VOLATILE_DIR/cass.pid`)
+      sleep_for 3
+      sh %(rm -rf #{cass_rootdir}/data)
     end
 
     task :execute do
@@ -66,7 +68,7 @@ def cass_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/common.rb b/ci/common.rb
@@ -15,8 +15,9 @@ def section(name)
   puts ''
 end
 
-# Initialize cache if in travis
-if ENV['TRAVIS']
+# Initialize cache if in travis and in our repository
+# (no cache for external contributors)
+if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
   cache = Cache.new({
     debug: ENV['DEBUG_CACHE'],
     s3: {
@@ -32,7 +33,7 @@ def section(name)
     task :before_install do |t|
       section('BEFORE_INSTALL')
       sh %(mkdir -p $VOLATILE_DIR)
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         cache.directories = ["#{ENV['HOME']}/embedded"]
         cache.setup
       end

diff --git a/ci/couchdb.rb b/ci/couchdb.rb
@@ -84,12 +84,17 @@ def couchdb_rootdir
       Rake::Task['ci:common:run_tests'].invoke(this_provides)
     end
 
-    task :before_cache => ['ci:common:before_cache']
+    task :before_cache => ['ci:common:before_cache'] do
+      # It's the pid file which changes eveytime,
+      # so let's actually cleanup before cache
+      Rake::Task['ci:couchdb:cleanup'].invoke
+    end
 
     task :cache => ['ci:common:cache']
 
     task :cleanup => ['ci:common:cleanup'] do
       sh %(#{couchdb_rootdir}/bin/couchdb -k)
+      sh %(rm -f #{couchdb_rootdir}/var/run/couchdb/couchdb.pid)
     end
 
     task :execute do
@@ -108,7 +113,7 @@ def couchdb_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/default.rb b/ci/default.rb
@@ -35,7 +35,7 @@
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/elasticsearch.rb b/ci/elasticsearch.rb
@@ -26,7 +26,9 @@ def es_rootdir
     end
 
     task :before_script => ['ci:common:before_script'] do
-      sh %(#{es_rootdir}/bin/elasticsearch -d)
+      pid = spawn %(#{es_rootdir}/bin/elasticsearch)
+      Process.detach(pid)
+      sh %(echo #{pid} > $VOLATILE_DIR/elasticsearch.pid)
       sleep_for 10
     end
 
@@ -37,12 +39,19 @@ def es_rootdir
       Rake::Task['ci:common:run_tests'].invoke(this_provides)
     end
 
-    task :before_cache => ['ci:common:before_cache']
+    task :before_cache => ['ci:common:before_cache'] do
+      Rake::Task['ci:elasticsearch:cleanup'].invoke
+    end
 
     task :cache => ['ci:common:cache']
 
-    task :cleanup => ['ci:common:cleanup']
-    # FIXME: stop elasticsearch
+    task :cleanup => ['ci:common:cleanup'] do
+      # FIXME: remove `|| true` when we drop support for ES 0.90.x
+      # (the only version spawning a process in background)
+      sh %(kill `cat $VOLATILE_DIR/elasticsearch.pid` || true)
+      sleep_for 1
+      sh %(rm -rf #{es_rootdir}/data || true)
+    end
 
     task :execute do
       exception = nil
@@ -60,7 +69,7 @@ def es_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/etcd.rb b/ci/etcd.rb
@@ -72,7 +72,7 @@ def etcd_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/fluentd.rb b/ci/fluentd.rb
@@ -46,7 +46,7 @@
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/gearman.rb b/ci/gearman.rb
@@ -64,7 +64,7 @@ def gearman_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/haproxy.rb b/ci/haproxy.rb
@@ -82,7 +82,7 @@ def haproxy_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/lighttpd.rb b/ci/lighttpd.rb
@@ -44,14 +44,17 @@ def lighttpd_rootdir
       Rake::Task['ci:common:run_tests'].invoke(this_provides)
     end
 
-    task :cleanup => ['ci:common:cleanup'] do
-      sh %(kill `cat $VOLATILE_DIR/lighttpd.pid`)
+    task :before_cache => ['ci:common:before_cache'] do
+      # Conf is regenerated at every run
+      sh %(rm -f #{lighttpd_rootdir}/lighttpd.conf)
     end
 
-    task :before_cache => ['ci:common:before_cache']
-
     task :cache => ['ci:common:cache']
 
+    task :cleanup => ['ci:common:cleanup'] do
+      sh %(kill `cat $VOLATILE_DIR/lighttpd.pid`)
+    end
+
     task :execute do
       exception = nil
       begin
@@ -68,7 +71,7 @@ def lighttpd_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/memcache.rb b/ci/memcache.rb
@@ -61,7 +61,7 @@ def memcache_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/mongo.rb b/ci/mongo.rb
@@ -98,7 +98,7 @@ def mongo_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/mysql.rb b/ci/mysql.rb
@@ -41,7 +41,7 @@
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/nginx.rb b/ci/nginx.rb
@@ -43,7 +43,10 @@ def nginx_rootdir
       Rake::Task['ci:common:run_tests'].invoke(this_provides)
     end
 
-    task :before_cache => ['ci:common:before_cache']
+    task :before_cache => ['ci:common:before_cache'] do
+      # Conf is regenerated at every run
+      sh %(rm -f #{nginx_rootdir}/conf/nginx.conf)
+    end
 
     task :cache => ['ci:common:cache']
 
@@ -67,7 +70,7 @@ def nginx_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/pgbouncer.rb b/ci/pgbouncer.rb
@@ -75,7 +75,7 @@ def pgb_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/phpfpm.rb b/ci/phpfpm.rb
@@ -86,7 +86,7 @@ def phpfpm_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/postgres.rb b/ci/postgres.rb
@@ -104,7 +104,7 @@ def pg_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/rabbitmq.rb b/ci/rabbitmq.rb
@@ -72,7 +72,7 @@ def rabbitmq_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/redis.rb b/ci/redis.rb
@@ -88,7 +88,7 @@ def redis_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/skeleton.rb b/ci/skeleton.rb
@@ -37,7 +37,7 @@
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/snmpd.rb b/ci/snmpd.rb
@@ -64,7 +64,7 @@ def snmpd_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/ssh.rb b/ci/ssh.rb
@@ -37,7 +37,7 @@
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/supervisord.rb b/ci/supervisord.rb
@@ -68,7 +68,7 @@ def supervisor_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/sysstat.rb b/ci/sysstat.rb
@@ -32,9 +32,9 @@ def sysstat_rootdir
     end
 
     task :before_script => ['ci:common:before_script'] do
-      # FIXME: probably not the cleanest way to go
       sh %(mkdir -p $INTEGRATIONS_DIR/bin)
-      sh %(cp #{sysstat_rootdir}/bin/mpstat $INTEGRATIONS_DIR/bin)
+      sh %(rm -f $INTEGRATIONS_DIR/bin/mpstat)
+      sh %(ln -s #{sysstat_rootdir}/bin/mpstat $INTEGRATIONS_DIR/bin/mpstat)
     end
 
     task :script => ['ci:common:script'] do
@@ -66,7 +66,7 @@ def sysstat_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end

diff --git a/ci/tomcat.rb b/ci/tomcat.rb
@@ -41,7 +41,11 @@ def tomcat_rootdir
       Rake::Task['ci:common:run_tests'].invoke(this_provides)
     end
 
-    task :before_cache => ['ci:common:before_cache']
+    task :before_cache => ['ci:common:before_cache'] do
+      # Regenerated at every run
+      sh %(rm -f #{tomcat_rootdir}/bin/setenv.sh)
+      sh %(rm -f #{tomcat_rootdir}/conf/server.xml)
+    end
 
     task :cache => ['ci:common:cache']
 
@@ -65,7 +69,7 @@ def tomcat_rootdir
         puts 'Cleaning up'
         Rake::Task["#{flavor.scope.path}:cleanup"].invoke
       end
-      if ENV['TRAVIS']
+      if ENV['TRAVIS'] && ENV['AWS_SECRET_ACCESS_KEY']
         %w(before_cache cache).each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end