From: Chris Lalancette<clalance(a)redhat.com> I modified Chris' patch to use a thread instead of fork. This pushes the communications with deltacloud into a thread which can take as long as it needs to complete the start. In the future we may want to put a retry in here too. You will notice in V2 we now set allow_concurrency = true in the postgres config. I'm a little worried that this means it may not work with other DB providers. V4: Fix whitespace errors Signed-off-by: Ian Main<imain(a)redhat.com> --- src/app/util/taskomatic.rb | 81 ++++++++++++++++++++++++++++++------------- src/config/database.pg | 3 ++ 2 files changed, 59 insertions(+), 25 deletions(-) diff --git a/src/app/util/taskomatic.rb b/src/app/util/taskomatic.rb index 84dabd0..2f08daa 100644 --- a/src/app/util/taskomatic.rb +++ b/src/app/util/taskomatic.rb @@ -26,27 +26,15 @@ module Taskomatic task.state = Task::STATE_PENDING task.save! - task.instance.provider_account = match.provider_account - task.instance.create_auth_key unless task.instance.instance_key - - dcloud_instance = create_dcloud_instance(task.instance, match) - - handle_dcloud_error(dcloud_instance) + create_dcloud_instance(task.instance, match) task.state = Task::STATE_RUNNING - task.save! - - Rails.logger.info "Task instance create completed with key #{dcloud_instance.id} and state #{dcloud_instance.state}" - task.instance.external_key = dcloud_instance.id - task.instance.state = dcloud_to_instance_state(dcloud_instance.state) - task.instance.save! rescue HttpException => ex task.failure_code = Task::FAILURE_PROVIDER_CONTACT_FAILED handle_create_instance_error(task, ex) rescue Exception => ex handle_create_instance_error(task, ex) ensure - task.instance.save! task.save! end end @@ -146,18 +134,61 @@ module Taskomatic end def self.create_dcloud_instance(instance, match) - client = match.provider_account.connect - - overrides = HardwareProfile.generate_override_property_values(instance.hardware_profile, match.hwp) - - client.create_instance(:image_id => match.provider_image.target_identifier, - :name => instance.name.tr("/", "-"), - :hwp_id => match.hwp.external_key, - :hwp_memory => overrides[:memory], - :hwp_cpu => overrides[:cpu], - :hwp_storage => overrides[:storage], - :realm_id => (match.realm.external_key rescue nil), - :keyname => (instance.instance_key.name)) + # because creating an instance can take a potentially long time (and + # creating multiple of them just prolongs this), we start a new thread + # that does this work. The new thread continues to run in the background + # and communicates the status back to the main UI via the database + + # This cleans up old DB connections. AR creates a new connection for each + # thread. This can leak FDs if you don't clean them up. Basically this + # ensures the previously used FD is cleaned up. + ActiveRecord::Base.connection_pool.clear_stale_cached_connections! + + Thread.new do + begin + # These all need to be reloaded because when you create a new thread + # AR creates a new connection and these objects become stale. + # + # Everything used here must be reloaded. + instance.reload + match.provider_account.reload + match.hwp.reload + match.realm.reload if match.realm + + instance.provider_account = match.provider_account + instance.create_auth_key unless instance.instance_key + + client = match.provider_account.connect + + overrides = HardwareProfile.generate_override_property_values(instance.hardware_profile, match.hwp) + + dcloud_instance = client.create_instance(:image_id => match.provider_image.target_identifier, + :name => instance.name.tr("/", "-"), + :hwp_id => match.hwp.external_key, + :hwp_memory => overrides[:memory], + :hwp_cpu => overrides[:cpu], + :hwp_storage => overrides[:storage], + :realm_id => (match.realm.external_key rescue nil), + :keyname => (instance.instance_key.name)) + + handle_dcloud_error(dcloud_instance) + + Rails.logger.info "Task instance create completed with key #{dcloud_instance.id} and state #{dcloud_instance.state}" + instance.external_key = dcloud_instance.id + instance.state = dcloud_to_instance_state(dcloud_instance.state) + rescue Exception => ex + # any sort of exception causes us to put the instance in CREATE_FAILED + # FIXME: if the exception is raised *after* the create_instance, this + # isn't true and will result in a rogue instance + Rails.logger.error ex.message + Rails.logger.error ex.backtrace.join("\n") + instance.state = Instance::STATE_CREATE_FAILED + raise ex + ensure + instance.save! + Thread.exit + end + end end def self.matches(instance) diff --git a/src/config/database.pg b/src/config/database.pg index 19fe4e6..72a3895 100644 --- a/src/config/database.pg +++ b/src/config/database.pg @@ -45,6 +45,7 @@ development: username: aeolus password: v23zj59an host: localhost + allow_concurrency: true min_messages: warning # Warning: The database defined as 'test' will be erased and @@ -56,6 +57,7 @@ test:&TEST username: aeolus password: v23zj59an host: localhost + allow_concurrency: true min_messages: warning production: @@ -63,6 +65,7 @@ production: database: conductor username: aeolus password: v23zj59an + allow_concurrency: true host: localhost cucumber: