master

DrydockLeaseUpdateWorker.php
View Options

			<?php

			/**
			* @task update Updating Leases
			* @task command Processing Commands
			* @task allocator Drydock Allocator
			* @task acquire Acquiring Leases
			* @task activate Activating Leases
			* @task release Releasing Leases
			* @task break Breaking Leases
			* @task destroy Destroying Leases
			*/
			final class DrydockLeaseUpdateWorker extends DrydockWorker {

			protected function doWork() {
			$lease_phid = $this->getTaskDataValue('leasePHID');

			$hash = PhabricatorHash::digestForIndex($lease_phid);
			$lock_key = 'drydock.lease:'.$hash;

			$lock = PhabricatorGlobalLock::newLock($lock_key)
			->lock(1);

			try {
			$lease = $this->loadLease($lease_phid);
			$this->handleUpdate($lease);
			} catch (Exception $ex) {
			$lock->unlock();
			$this->flushDrydockTaskQueue();
			throw $ex;
			}

			$lock->unlock();
			}


			/* -( Updating Leases )---------------------------------------------------- */


			/**
			* @task update
			*/
			private function handleUpdate(DrydockLease $lease) {
			try {
			$this->updateLease($lease);
			} catch (DrydockAcquiredBrokenResourceException $ex) {
			// If this lease acquired a resource but failed to activate, we don't
			// need to break the lease. We can throw it back in the pool and let
			// it take another shot at acquiring a new resource.

			// Before we throw it back, release any locks the lease is holding.
			DrydockSlotLock::releaseLocks($lease->getPHID());

			$lease
			->setStatus(DrydockLeaseStatus::STATUS_PENDING)
			->setResourcePHID(null)
			->save();

			$lease->logEvent(
			DrydockLeaseReacquireLogType::LOGCONST,
			array(
			'class' => get_class($ex),
			'message' => $ex->getMessage(),
			));

			$this->yieldLease($lease, $ex);
			} catch (Exception $ex) {
			if ($this->isTemporaryException($ex)) {
			$this->yieldLease($lease, $ex);
			} else {
			$this->breakLease($lease, $ex);
			}
			}
			}


			/**
			* @task update
			*/
			private function updateLease(DrydockLease $lease) {
			$this->processLeaseCommands($lease);

			$lease_status = $lease->getStatus();
			switch ($lease_status) {
			case DrydockLeaseStatus::STATUS_PENDING:
			$this->executeAllocator($lease);
			break;
			case DrydockLeaseStatus::STATUS_ACQUIRED:
			$this->activateLease($lease);
			break;
			case DrydockLeaseStatus::STATUS_ACTIVE:
			// Nothing to do.
			break;
			case DrydockLeaseStatus::STATUS_RELEASED:
			case DrydockLeaseStatus::STATUS_BROKEN:
			$this->destroyLease($lease);
			break;
			case DrydockLeaseStatus::STATUS_DESTROYED:
			break;
			}

			$this->yieldIfExpiringLease($lease);
			}


			/**
			* @task update
			*/
			private function yieldLease(DrydockLease $lease, Exception $ex) {
			$duration = $this->getYieldDurationFromException($ex);

			$lease->logEvent(
			DrydockLeaseActivationYieldLogType::LOGCONST,
			array(
			'duration' => $duration,
			));

			throw new PhabricatorWorkerYieldException($duration);
			}


			/* -( Processing Commands )------------------------------------------------ */


			/**
			* @task command
			*/
			private function processLeaseCommands(DrydockLease $lease) {
			if (!$lease->canReceiveCommands()) {
			return;
			}

			$this->checkLeaseExpiration($lease);

			$commands = $this->loadCommands($lease->getPHID());
			foreach ($commands as $command) {
			if (!$lease->canReceiveCommands()) {
			break;
			}

			$this->processLeaseCommand($lease, $command);

			$command
			->setIsConsumed(true)
			->save();
			}
			}


			/**
			* @task command
			*/
			private function processLeaseCommand(
			DrydockLease $lease,
			DrydockCommand $command) {
			switch ($command->getCommand()) {
			case DrydockCommand::COMMAND_RELEASE:
			$this->releaseLease($lease);
			break;
			}
			}


			/* -( Drydock Allocator )-------------------------------------------------- */


			/**
			* Find or build a resource which can satisfy a given lease request, then
			* acquire the lease.
			*
			* @param DrydockLease $lease Requested lease.
			* @return void
			* @task allocator
			*/
			private function executeAllocator(DrydockLease $lease) {
			$blueprints = $this->loadBlueprintsForAllocatingLease($lease);

			// If we get nothing back, that means no blueprint is defined which can
			// ever build the requested resource. This is a permanent failure, since
			// we don't expect to succeed no matter how many times we try.
			if (!$blueprints) {
			throw new PhabricatorWorkerPermanentFailureException(
			pht(
			'No active Drydock blueprint exists which can ever allocate a '.
			'resource for lease "%s".',
			$lease->getPHID()));
			}

			// First, try to find a suitable open resource which we can acquire a new
			// lease on.
			$resources = $this->loadAcquirableResourcesForLease($blueprints, $lease);

			list($free_resources, $used_resources) = $this->partitionResources(
			$lease,
			$resources);

			$resource = $this->leaseAnyResource($lease, $free_resources);
			if ($resource) {
			return $resource;
			}

			// We're about to try creating a resource. If we're already creating
			// something, just yield until that resolves.

			$this->yieldForPendingResources($lease);

			// We haven't been able to lease an existing resource yet, so now we try to
			// create one. We may still have some less-desirable "used" resources that
			// we'll sometimes try to lease later if we fail to allocate a new resource.

			$resource = $this->newLeasedResource($lease, $blueprints);
			if ($resource) {
			return $resource;
			}

			// We haven't been able to lease a desirable "free" resource or create a
			// new resource. Try to lease a "used" resource.

			$resource = $this->leaseAnyResource($lease, $used_resources);
			if ($resource) {
			return $resource;
			}

			// If this lease has already triggered a reclaim, just yield and wait for
			// it to resolve.
			$this->yieldForReclaimingResources($lease);

			// Try to reclaim a resource. This will yield if it reclaims something.
			$this->reclaimAnyResource($lease, $blueprints);

			// We weren't able to lease, create, or reclaim any resources. We just have
			// to wait for resources to become available.

			$lease->logEvent(
			DrydockLeaseWaitingForResourcesLogType::LOGCONST,
			array(
			'blueprintPHIDs' => mpull($blueprints, 'getPHID'),
			));

			throw new PhabricatorWorkerYieldException(15);
			}

			private function reclaimAnyResource(DrydockLease $lease, array $blueprints) {
			assert_instances_of($blueprints, 'DrydockBlueprint');

			$blueprints = $this->rankBlueprints($blueprints, $lease);

			// Try to actively reclaim unused resources. If we succeed, jump back
			// into the queue in an effort to claim it.

			foreach ($blueprints as $blueprint) {
			$reclaimed = $this->reclaimResources($blueprint, $lease);
			if ($reclaimed) {

			$lease->logEvent(
			DrydockLeaseReclaimLogType::LOGCONST,
			array(
			'resourcePHIDs' => array($reclaimed->getPHID()),
			));

			// Yield explicitly here: we'll be awakened when the resource is
			// reclaimed.

			throw new PhabricatorWorkerYieldException(15);
			}
			}
			}

			private function yieldForPendingResources(DrydockLease $lease) {
			// See T13677. If this lease has already triggered the allocation of
			// one or more resources and they are still pending, just yield and
			// wait for them.

			$viewer = $this->getViewer();

			$phids = $lease->getAllocatedResourcePHIDs();
			if (!$phids) {
			return null;
			}

			$resources = id(new DrydockResourceQuery())
			->setViewer($viewer)
			->withPHIDs($phids)
			->withStatuses(
			array(
			DrydockResourceStatus::STATUS_PENDING,
			))
			->setLimit(1)
			->execute();
			if (!$resources) {
			return;
			}

			$lease->logEvent(
			DrydockLeaseWaitingForActivationLogType::LOGCONST,
			array(
			'resourcePHIDs' => mpull($resources, 'getPHID'),
			));

			throw new PhabricatorWorkerYieldException(15);
			}

			private function yieldForReclaimingResources(DrydockLease $lease) {
			$viewer = $this->getViewer();

			$phids = $lease->getReclaimedResourcePHIDs();
			if (!$phids) {
			return;
			}

			$resources = id(new DrydockResourceQuery())
			->setViewer($viewer)
			->withPHIDs($phids)
			->withStatuses(
			array(
			DrydockResourceStatus::STATUS_ACTIVE,
			DrydockResourceStatus::STATUS_RELEASED,
			))
			->setLimit(1)
			->execute();
			if (!$resources) {
			return;
			}

			$lease->logEvent(
			DrydockLeaseWaitingForReclamationLogType::LOGCONST,
			array(
			'resourcePHIDs' => mpull($resources, 'getPHID'),
			));

			throw new PhabricatorWorkerYieldException(15);
			}

			private function newLeasedResource(
			DrydockLease $lease,
			array $blueprints) {
			assert_instances_of($blueprints, 'DrydockBlueprint');

			$usable_blueprints = $this->removeOverallocatedBlueprints(
			$blueprints,
			$lease);

			// If we get nothing back here, some blueprint claims it can eventually
			// satisfy the lease, just not right now. This is a temporary failure,
			// and we expect allocation to succeed eventually.

			// Return, try to lease a "used" resource, and continue from there.

			if (!$usable_blueprints) {
			return null;
			}

			$usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease);

			$new_resources = $this->newResources($lease, $usable_blueprints);
			if (!$new_resources) {
			// If we were unable to create any new resources, return and
			// try to lease a "used" resource.
			return null;
			}

			$new_resources = $this->removeUnacquirableResources(
			$new_resources,
			$lease);
			if (!$new_resources) {
			// If we make it here, we just built a resource but aren't allowed
			// to acquire it. We expect this to happen if the resource prevents
			// acquisition until it activates, which is common when a resource
			// needs to perform setup steps.

			// Explicitly yield and wait for activation, since we don't want to
			// lease a "used" resource.

			throw new PhabricatorWorkerYieldException(15);
			}

			$resource = $this->leaseAnyResource($lease, $new_resources);
			if ($resource) {
			return $resource;
			}

			// We may not be able to lease a resource even if we just built it:
			// another process may snatch it up before we can lease it. This should
			// be rare, but is not concerning. Just try to build another resource.

			// We likely could try to build the next resource immediately, but err on
			// the side of caution and yield for now, at least until this code is
			// better vetted.

			throw new PhabricatorWorkerYieldException(15);
			}

			private function partitionResources(
			DrydockLease $lease,
			array $resources) {

			assert_instances_of($resources, 'DrydockResource');
			$viewer = $this->getViewer();

			$lease_statuses = array(
			DrydockLeaseStatus::STATUS_PENDING,
			DrydockLeaseStatus::STATUS_ACQUIRED,
			DrydockLeaseStatus::STATUS_ACTIVE,
			);

			// Partition resources into "free" resources (which we can try to lease
			// immediately) and "used" resources, which we can only to lease after we
			// fail to allocate a new resource.

			// "Free" resources are unleased and/or prefer reuse over allocation.
			// "Used" resources are leased and prefer allocation over reuse.

			$free_resources = array();
			$used_resources = array();

			foreach ($resources as $resource) {
			$blueprint = $resource->getBlueprint();

			if (!$blueprint->shouldAllocateSupplementalResource($resource, $lease)) {
			$free_resources[] = $resource;
			continue;
			}

			$leases = id(new DrydockLeaseQuery())
			->setViewer($viewer)
			->withResourcePHIDs(array($resource->getPHID()))
			->withStatuses($lease_statuses)
			->setLimit(1)
			->execute();
			if (!$leases) {
			$free_resources[] = $resource;
			continue;
			}

			$used_resources[] = $resource;
			}

			return array($free_resources, $used_resources);
			}

			private function newResources(
			DrydockLease $lease,
			array $blueprints) {
			assert_instances_of($blueprints, 'DrydockBlueprint');

			$resources = array();
			$exceptions = array();
			foreach ($blueprints as $blueprint) {
			$caught = null;
			try {
			$resources[] = $this->allocateResource($blueprint, $lease);

			// Bail after allocating one resource, we don't need any more than
			// this.
			break;
			} catch (Exception $ex) {
			$caught = $ex;
			} catch (Throwable $ex) {
			$caught = $ex;
			}

			if ($caught) {
			// This failure is not normally expected, so log it. It can be
			// caused by something mundane and recoverable, however (see below
			// for discussion).

			// We log to the blueprint separately from the log to the lease:
			// the lease is not attached to a blueprint yet so the lease log
			// will not show up on the blueprint; more than one blueprint may
			// fail; and the lease is not really impacted (and won't log) if at
			// least one blueprint actually works.

			$blueprint->logEvent(
			DrydockResourceAllocationFailureLogType::LOGCONST,
			array(
			'class' => get_class($caught),
			'message' => $caught->getMessage(),
			));

			$exceptions[] = $caught;
			}
			}

			if (!$resources) {
			// If one or more blueprints claimed that they would be able to allocate
			// resources but none are actually able to allocate resources, log the
			// failure and yield so we try again soon.

			// This can happen if some unexpected issue occurs during allocation
			// (for example, a call to build a VM fails for some reason) or if we
			// raced another allocator and the blueprint is now full.

			$ex = new PhutilAggregateException(
			pht(
			'All blueprints failed to allocate a suitable new resource when '.
			'trying to allocate lease ("%s").',
			$lease->getPHID()),
			$exceptions);

			$lease->logEvent(
			DrydockLeaseAllocationFailureLogType::LOGCONST,
			array(
			'class' => get_class($ex),
			'message' => $ex->getMessage(),
			));

			return null;
			}

			return $resources;
			}


			private function leaseAnyResource(
			DrydockLease $lease,
			array $resources) {
			assert_instances_of($resources, 'DrydockResource');

			if (!$resources) {
			return null;
			}

			$resources = $this->rankResources($resources, $lease);

			$exceptions = array();
			$yields = array();

			$allocated = null;
			foreach ($resources as $resource) {
			try {
			$this->acquireLease($resource, $lease);
			$allocated = $resource;
			break;
			} catch (DrydockResourceLockException $ex) {
			// We need to lock the resource to actually acquire it. If we aren't
			// able to acquire the lock quickly enough, we can yield and try again
			// later.
			$yields[] = $ex;
			} catch (DrydockSlotLockException $ex) {
			// This also just indicates we ran into some kind of contention,
			// probably from another lease. Just yield.
			$yields[] = $ex;
			} catch (DrydockAcquiredBrokenResourceException $ex) {
			// If a resource was reclaimed or destroyed by the time we actually
			// got around to acquiring it, we just got unlucky.
			$yields[] = $ex;
			} catch (PhabricatorWorkerYieldException $ex) {
			// We can be told to yield, particularly by the supplemental allocator
			// trying to give us a supplemental resource.
			$yields[] = $ex;
			} catch (Exception $ex) {
			$exceptions[] = $ex;
			}
			}

			if ($allocated) {
			return $allocated;
			}

			if ($yields) {
			throw new PhabricatorWorkerYieldException(15);
			}

			throw new PhutilAggregateException(
			pht(
			'Unable to acquire lease "%s" on any resource.',
			$lease->getPHID()),
			$exceptions);
			}


			/**
			* Get all the concrete @{class:DrydockBlueprint}s which can possibly
			* build a resource to satisfy a lease.
			*
			* @param DrydockLease $lease Requested lease.
			* @return list<DrydockBlueprint> List of qualifying blueprints.
			* @task allocator
			*/
			private function loadBlueprintsForAllocatingLease(
			DrydockLease $lease) {
			$viewer = $this->getViewer();

			$impls = DrydockBlueprintImplementation::getAllForAllocatingLease($lease);
			if (!$impls) {
			return array();
			}

			$blueprint_phids = $lease->getAllowedBlueprintPHIDs();
			if (!$blueprint_phids) {
			$lease->logEvent(DrydockLeaseNoBlueprintsLogType::LOGCONST);
			return array();
			}

			$query = id(new DrydockBlueprintQuery())
			->setViewer($viewer)
			->withPHIDs($blueprint_phids)
			->withBlueprintClasses(array_keys($impls))
			->withDisabled(false);

			// The Drydock application itself is allowed to authorize anything. This
			// is primarily used for leases generated by CLI administrative tools.
			$drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();

			$authorizing_phid = $lease->getAuthorizingPHID();
			if ($authorizing_phid != $drydock_phid) {
			$blueprints = id(clone $query)
			->withAuthorizedPHIDs(array($authorizing_phid))
			->execute();
			if (!$blueprints) {
			// If we didn't hit any blueprints, check if this is an authorization
			// problem: re-execute the query without the authorization constraint.
			// If the second query hits blueprints, the overall configuration is
			// fine but this is an authorization problem. If the second query also
			// comes up blank, this is some other kind of configuration issue so
			// we fall through to the default pathway.
			$all_blueprints = $query->execute();
			if ($all_blueprints) {
			$lease->logEvent(
			DrydockLeaseNoAuthorizationsLogType::LOGCONST,
			array(
			'authorizingPHID' => $authorizing_phid,
			));
			return array();
			}
			}
			} else {
			$blueprints = $query->execute();
			}

			$keep = array();
			foreach ($blueprints as $key => $blueprint) {
			if (!$blueprint->canEverAllocateResourceForLease($lease)) {
			continue;
			}

			$keep[$key] = $blueprint;
			}

			return $keep;
			}


			/**
			* Load a list of all resources which a given lease can possibly be
			* allocated against.
			*
			* @param list<DrydockBlueprint> $blueprints Blueprints which may produce
			* suitable resources.
			* @param DrydockLease $lease Requested lease.
			* @return list<DrydockResource> Resources which may be able to allocate
			* the lease.
			* @task allocator
			*/
			private function loadAcquirableResourcesForLease(
			array $blueprints,
			DrydockLease $lease) {
			assert_instances_of($blueprints, 'DrydockBlueprint');
			$viewer = $this->getViewer();

			$resources = id(new DrydockResourceQuery())
			->setViewer($viewer)
			->withBlueprintPHIDs(mpull($blueprints, 'getPHID'))
			->withTypes(array($lease->getResourceType()))
			->withStatuses(
			array(
			DrydockResourceStatus::STATUS_ACTIVE,
			))
			->execute();

			return $this->removeUnacquirableResources($resources, $lease);
			}


			/**
			* Remove resources which can not be acquired by a given lease from a list.
			*
			* @param list<DrydockResource> $resources Candidate resources.
			* @param DrydockLease $lease Acquiring lease.
			* @return list<DrydockResource> Resources which the lease may be able to
			* acquire.
			* @task allocator
			*/
			private function removeUnacquirableResources(
			array $resources,
			DrydockLease $lease) {
			$keep = array();
			foreach ($resources as $key => $resource) {
			$blueprint = $resource->getBlueprint();

			if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) {
			continue;
			}

			$keep[$key] = $resource;
			}

			return $keep;
			}


			/**
			* Remove blueprints which are too heavily allocated to build a resource for
			* a lease from a list of blueprints.
			*
			* @param list<DrydockBlueprint> $blueprints List of blueprints.
			* @return list<DrydockBlueprint> $lease List with blueprints that can not
			* allocate a resource for the lease right now removed.
			* @task allocator
			*/
			private function removeOverallocatedBlueprints(
			array $blueprints,
			DrydockLease $lease) {
			assert_instances_of($blueprints, 'DrydockBlueprint');

			$keep = array();

			foreach ($blueprints as $key => $blueprint) {
			if (!$blueprint->canAllocateResourceForLease($lease)) {
			continue;
			}

			$keep[$key] = $blueprint;
			}

			return $keep;
			}


			/**
			* Rank blueprints by suitability for building a new resource for a
			* particular lease.
			*
			* @param list<DrydockBlueprint> $blueprints List of blueprints.
			* @param DrydockLease $lease Requested lease.
			* @return list<DrydockBlueprint> Ranked list of blueprints.
			* @task allocator
			*/
			private function rankBlueprints(array $blueprints, DrydockLease $lease) {
			assert_instances_of($blueprints, 'DrydockBlueprint');

			// TODO: Implement improvements to this ranking algorithm if they become
			// available.
			shuffle($blueprints);

			return $blueprints;
			}


			/**
			* Rank resources by suitability for allocating a particular lease.
			*
			* @param list<DrydockResource> $resources List of resources.
			* @param DrydockLease $lease Requested lease.
			* @return list<DrydockResource> Ranked list of resources.
			* @task allocator
			*/
			private function rankResources(array $resources, DrydockLease $lease) {
			assert_instances_of($resources, 'DrydockResource');

			// TODO: Implement improvements to this ranking algorithm if they become
			// available.
			shuffle($resources);

			return $resources;
			}


			/**
			* Perform an actual resource allocation with a particular blueprint.
			*
			* @param DrydockBlueprint $blueprint The blueprint to allocate a resource
			* from.
			* @param DrydockLease $lease Requested lease.
			* @return DrydockResource Allocated resource.
			* @task allocator
			*/
			private function allocateResource(
			DrydockBlueprint $blueprint,
			DrydockLease $lease) {
			$resource = $blueprint->allocateResource($lease);
			$this->validateAllocatedResource($blueprint, $resource, $lease);

			// If this resource was allocated as a pending resource, queue a task to
			// activate it.
			if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) {

			$lease->addAllocatedResourcePHIDs(
			array(
			$resource->getPHID(),
			));
			$lease->save();

			PhabricatorWorker::scheduleTask(
			'DrydockResourceUpdateWorker',
			array(
			'resourcePHID' => $resource->getPHID(),

			// This task will generally yield while the resource activates, so
			// wake it back up once the resource comes online. Most of the time,
			// we'll be able to lease the newly activated resource.
			'awakenOnActivation' => array(
			$this->getCurrentWorkerTaskID(),
			),
			),
			array(
			'objectPHID' => $resource->getPHID(),
			));
			}

			return $resource;
			}


			/**
			* Check that the resource a blueprint allocated is roughly the sort of
			* object we expect.
			*
			* @param DrydockBlueprint $blueprint Blueprint which built the resource.
			* @param wild $resource Thing which the blueprint claims is a valid
			* resource.
			* @param DrydockLease $lease Lease the resource was allocated for.
			* @return void
			* @task allocator
			*/
			private function validateAllocatedResource(
			DrydockBlueprint $blueprint,
			$resource,
			DrydockLease $lease) {

			if (!($resource instanceof DrydockResource)) {
			throw new Exception(
			pht(
			'Blueprint "%s" (of type "%s") is not properly implemented: %s must '.
			'return an object of type %s or throw, but returned something else.',
			$blueprint->getBlueprintName(),
			$blueprint->getClassName(),
			'allocateResource()',
			'DrydockResource'));
			}

			if (!$resource->isAllocatedResource()) {
			throw new Exception(
			pht(
			'Blueprint "%s" (of type "%s") is not properly implemented: %s '.
			'must actually allocate the resource it returns.',
			$blueprint->getBlueprintName(),
			$blueprint->getClassName(),
			'allocateResource()'));
			}

			$resource_type = $resource->getType();
			$lease_type = $lease->getResourceType();

			if ($resource_type !== $lease_type) {
			throw new Exception(
			pht(
			'Blueprint "%s" (of type "%s") is not properly implemented: it '.
			'built a resource of type "%s" to satisfy a lease requesting a '.
			'resource of type "%s".',
			$blueprint->getBlueprintName(),
			$blueprint->getClassName(),
			$resource_type,
			$lease_type));
			}
			}

			private function reclaimResources(
			DrydockBlueprint $blueprint,
			DrydockLease $lease) {
			$viewer = $this->getViewer();

			$resources = id(new DrydockResourceQuery())
			->setViewer($viewer)
			->withBlueprintPHIDs(array($blueprint->getPHID()))
			->withStatuses(
			array(
			DrydockResourceStatus::STATUS_ACTIVE,
			))
			->execute();

			// TODO: We could be much smarter about this and try to release long-unused
			// resources, resources with many similar copies, old resources, resources
			// that are cheap to rebuild, etc.
			shuffle($resources);

			foreach ($resources as $resource) {
			if ($this->canReclaimResource($resource)) {
			$this->reclaimResource($resource, $lease);
			return $resource;
			}
			}

			return null;
			}


			/* -( Acquiring Leases )--------------------------------------------------- */


			/**
			* Perform an actual lease acquisition on a particular resource.
			*
			* @param DrydockResource $resource Resource to acquire a lease on.
			* @param DrydockLease $lease Lease to acquire.
			* @return void
			* @task acquire
			*/
			private function acquireLease(
			DrydockResource $resource,
			DrydockLease $lease) {

			$blueprint = $resource->getBlueprint();
			$blueprint->acquireLease($resource, $lease);

			$this->validateAcquiredLease($blueprint, $resource, $lease);

			// If this lease has been acquired but not activated, queue a task to
			// activate it.
			if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) {
			$this->queueTask(
			__CLASS__,
			array(
			'leasePHID' => $lease->getPHID(),
			),
			array(
			'objectPHID' => $lease->getPHID(),
			));
			}
			}


			/**
			* Make sure that a lease was really acquired properly.
			*
			* @param DrydockBlueprint $blueprint Blueprint which created the resource.
			* @param DrydockResource $resource Resource which was acquired.
			* @param DrydockLease $lease The lease which was supposedly acquired.
			* @return void
			* @task acquire
			*/
			private function validateAcquiredLease(
			DrydockBlueprint $blueprint,
			DrydockResource $resource,
			DrydockLease $lease) {

			if (!$lease->isAcquiredLease()) {
			throw new Exception(
			pht(
			'Blueprint "%s" (of type "%s") is not properly implemented: it '.
			'returned from "%s" without acquiring a lease.',
			$blueprint->getBlueprintName(),
			$blueprint->getClassName(),
			'acquireLease()'));
			}

			$lease_phid = $lease->getResourcePHID();
			$resource_phid = $resource->getPHID();

			if ($lease_phid !== $resource_phid) {
			throw new Exception(
			pht(
			'Blueprint "%s" (of type "%s") is not properly implemented: it '.
			'returned from "%s" with a lease acquired on the wrong resource.',
			$blueprint->getBlueprintName(),
			$blueprint->getClassName(),
			'acquireLease()'));
			}
			}


			/* -( Activating Leases )-------------------------------------------------- */


			/**
			* @task activate
			*/
			private function activateLease(DrydockLease $lease) {
			$resource = $lease->getResource();
			if (!$resource) {
			throw new Exception(
			pht('Trying to activate lease with no resource.'));
			}

			$resource_status = $resource->getStatus();

			if ($resource_status == DrydockResourceStatus::STATUS_PENDING) {
			throw new PhabricatorWorkerYieldException(15);
			}

			if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) {
			throw new DrydockAcquiredBrokenResourceException(
			pht(
			'Trying to activate lease ("%s") on a resource ("%s") in '.
			'the wrong status ("%s").',
			$lease->getPHID(),
			$resource->getPHID(),
			$resource_status));
			}

			// NOTE: We can race resource destruction here. Between the time we
			// performed the read above and now, the resource might have closed, so
			// we may activate leases on dead resources. At least for now, this seems
			// fine: a resource dying right before we activate a lease on it should not
			// be distinguishable from a resource dying right after we activate a lease
			// on it. We end up with an active lease on a dead resource either way, and
			// can not prevent resources dying from lightning strikes.

			$blueprint = $resource->getBlueprint();
			$blueprint->activateLease($resource, $lease);
			$this->validateActivatedLease($blueprint, $resource, $lease);
			}

			/**
			* @task activate
			*/
			private function validateActivatedLease(
			DrydockBlueprint $blueprint,
			DrydockResource $resource,
			DrydockLease $lease) {

			if (!$lease->isActivatedLease()) {
			throw new Exception(
			pht(
			'Blueprint "%s" (of type "%s") is not properly implemented: it '.
			'returned from "%s" without activating a lease.',
			$blueprint->getBlueprintName(),
			$blueprint->getClassName(),
			'acquireLease()'));
			}

			}


			/* -( Releasing Leases )--------------------------------------------------- */


			/**
			* @task release
			*/
			private function releaseLease(DrydockLease $lease) {
			$lease
			->setStatus(DrydockLeaseStatus::STATUS_RELEASED)
			->save();

			$lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST);

			$resource = $lease->getResource();
			if ($resource) {
			$blueprint = $resource->getBlueprint();
			$blueprint->didReleaseLease($resource, $lease);
			}

			$this->destroyLease($lease);
			}


			/* -( Breaking Leases )---------------------------------------------------- */


			/**
			* @task break
			*/
			protected function breakLease(DrydockLease $lease, Exception $ex) {
			switch ($lease->getStatus()) {
			case DrydockLeaseStatus::STATUS_BROKEN:
			case DrydockLeaseStatus::STATUS_RELEASED:
			case DrydockLeaseStatus::STATUS_DESTROYED:
			throw new PhutilProxyException(
			pht(
			'Unexpected failure while destroying lease ("%s").',
			$lease->getPHID()),
			$ex);
			}

			$lease
			->setStatus(DrydockLeaseStatus::STATUS_BROKEN)
			->save();

			$lease->logEvent(
			DrydockLeaseActivationFailureLogType::LOGCONST,
			array(
			'class' => get_class($ex),
			'message' => $ex->getMessage(),
			));

			$lease->awakenTasks();

			$this->queueTask(
			__CLASS__,
			array(
			'leasePHID' => $lease->getPHID(),
			),
			array(
			'objectPHID' => $lease->getPHID(),
			));

			throw new PhabricatorWorkerPermanentFailureException(
			pht(
			'Permanent failure while activating lease ("%s"): %s',
			$lease->getPHID(),
			$ex->getMessage()));
			}


			/* -( Destroying Leases )-------------------------------------------------- */


			/**
			* @task destroy
			*/
			private function destroyLease(DrydockLease $lease) {
			$resource = $lease->getResource();

			if ($resource) {
			$blueprint = $resource->getBlueprint();
			$blueprint->destroyLease($resource, $lease);
			}

			DrydockSlotLock::releaseLocks($lease->getPHID());

			$lease
			->setStatus(DrydockLeaseStatus::STATUS_DESTROYED)
			->save();

			$lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST);

			$lease->awakenTasks();
			}

			}

Owner Packages

No Owners

phorge/src/applications/drydock/worker/DrydockLeaseUpdateWorker.phpe81eff68e581HEAD -> master

DrydockLeaseUpdateWorker.phpView Options

Owner Packages

phorge/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
e81eff68e581HEAD -> master

DrydockLeaseUpdateWorker.php
View Options